{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Edgeprobe Aggregate Analysis (for ICLR camera-ready)\n",
    "\n",
    "This is the main analysis notebook for [What do you learn from context? Probing for sentence structure in contextualized word representations](https://openreview.net/forum?id=SJzSgnRcKX), a.k.a. \"the edge probing paper.\"\n",
    "\n",
    "This notebook is intended to be run on the output of the [`analyze_runs.py`](analyze_runs.py) script; run that on a folder of experiments to produce a `scores.tsv` file that can be loaded here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os, re, json\n",
    "from importlib import reload\n",
    "import itertools\n",
    "import collections\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    }\n",
       "    finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        root._bokeh_is_loading--;\n",
       "        if (root._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"1001\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.0.4.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n    }\n    finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.info(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(js_urls, callback) {\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = js_urls.length;\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var s = document.createElement('script');\n      s.src = url;\n      s.async = false;\n      s.onreadystatechange = s.onload = function() {\n        root._bokeh_is_loading--;\n        if (root._bokeh_is_loading === 0) {\n          console.log(\"Bokeh: all BokehJS libraries loaded\");\n          run_callbacks()\n        }\n      };\n      s.onerror = function() {\n        console.warn(\"failed to load library \" + url);\n      };\n      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.getElementsByTagName(\"head\")[0].appendChild(s);\n    }\n  };var element = document.getElementById(\"1001\");\n  if (element == null) {\n    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n    return false;\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.0.4.min.js\"];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.css\");\n    }\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(js_urls, function() {\n      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import bokeh\n",
    "import bokeh.plotting as bp\n",
    "bp.output_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<module 'analysis' from '/nfs/jsalt/home/iftenney/jiant_test/probing/analysis.py'>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import analysis\n",
    "reload(analysis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('glove', '#ffbb78'),\n",
       "             ('glove-cnn1', '#ff7f0e'),\n",
       "             ('glove-cnn2', '#ff7f0e'),\n",
       "             ('cove', '#ff9896'),\n",
       "             ('cove-cnn1', '#d62728'),\n",
       "             ('cove-cnn2', '#d62728'),\n",
       "             ('elmo-chars', '#9edae5'),\n",
       "             ('elmo-chars-cnn1', '#17becf'),\n",
       "             ('elmo-chars-cnn2', '#17becf'),\n",
       "             ('elmo-ortho', '#c5b0d5'),\n",
       "             ('elmo-ortho-cnn1', '#9467bd'),\n",
       "             ('elmo-ortho-cnn2', '#9467bd'),\n",
       "             ('elmo-full', '#aec7e8'),\n",
       "             ('elmo-full-cnn1', '#1f77b4'),\n",
       "             ('elmo-full-cnn2', '#1f77b4'),\n",
       "             ('openai-lex', '#dbdb8d'),\n",
       "             ('openai-lex-cnn1', '#bcbd22'),\n",
       "             ('openai-lex-cnn2', '#bcbd22'),\n",
       "             ('openai-cat', '#98df8a'),\n",
       "             ('openai-cat-cnn1', '#2ca02c'),\n",
       "             ('openai-cat-cnn2', '#2ca02c'),\n",
       "             ('openai-mix', '#f7b6d2'),\n",
       "             ('openai-mix-cnn1', '#e377c2'),\n",
       "             ('openai-mix-cnn2', '#e377c2'),\n",
       "             ('openai', '#98df8a'),\n",
       "             ('openai-cnn1', '#2ca02c'),\n",
       "             ('openai-cnn2', '#2ca02c'),\n",
       "             ('openai-bwb', '#f7b6d2'),\n",
       "             ('openai-bwb-cnn1', '#e377c2'),\n",
       "             ('openai-bwb-cnn2', '#e377c2'),\n",
       "             ('train-chars', '#c49c94'),\n",
       "             ('train-chars-cnn1', '#8c564b'),\n",
       "             ('train-chars-cnn2', '#8c564b'),\n",
       "             ('bert-base-uncased-lex', '#dbdb8d'),\n",
       "             ('bert-base-uncased-lex-cnn1', '#bcbd22'),\n",
       "             ('bert-base-uncased-lex-cnn2', '#bcbd22'),\n",
       "             ('bert-base-uncased-cat', '#98df8a'),\n",
       "             ('bert-base-uncased-cat-cnn1', '#2ca02c'),\n",
       "             ('bert-base-uncased-cat-cnn2', '#2ca02c'),\n",
       "             ('bert-base-uncased-mix', '#f7b6d2'),\n",
       "             ('bert-base-uncased-mix-cnn1', '#e377c2'),\n",
       "             ('bert-base-uncased-mix-cnn2', '#e377c2'),\n",
       "             ('bert-base-cased-lex', '#dbdb8d'),\n",
       "             ('bert-base-cased-lex-cnn1', '#bcbd22'),\n",
       "             ('bert-base-cased-lex-cnn2', '#bcbd22'),\n",
       "             ('bert-base-cased-cat', '#98df8a'),\n",
       "             ('bert-base-cased-cat-cnn1', '#2ca02c'),\n",
       "             ('bert-base-cased-cat-cnn2', '#2ca02c'),\n",
       "             ('bert-base-cased-mix', '#f7b6d2'),\n",
       "             ('bert-base-cased-mix-cnn1', '#e377c2'),\n",
       "             ('bert-base-cased-mix-cnn2', '#e377c2'),\n",
       "             ('bert-large-uncased-lex', '#dbdb8d'),\n",
       "             ('bert-large-uncased-lex-cnn1', '#bcbd22'),\n",
       "             ('bert-large-uncased-lex-cnn2', '#bcbd22'),\n",
       "             ('bert-large-uncased-cat', '#98df8a'),\n",
       "             ('bert-large-uncased-cat-cnn1', '#2ca02c'),\n",
       "             ('bert-large-uncased-cat-cnn2', '#2ca02c'),\n",
       "             ('bert-large-uncased-mix', '#f7b6d2'),\n",
       "             ('bert-large-uncased-mix-cnn1', '#e377c2'),\n",
       "             ('bert-large-uncased-mix-cnn2', '#e377c2'),\n",
       "             ('bert-large-cased-lex', '#dbdb8d'),\n",
       "             ('bert-large-cased-lex-cnn1', '#bcbd22'),\n",
       "             ('bert-large-cased-lex-cnn2', '#bcbd22'),\n",
       "             ('bert-large-cased-cat', '#98df8a'),\n",
       "             ('bert-large-cased-cat-cnn1', '#2ca02c'),\n",
       "             ('bert-large-cased-cat-cnn2', '#2ca02c'),\n",
       "             ('bert-large-cased-mix', '#f7b6d2'),\n",
       "             ('bert-large-cased-mix-cnn1', '#e377c2'),\n",
       "             ('bert-large-cased-mix-cnn2', '#e377c2')])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tasks = [\n",
    "    \"pos-ontonotes\",\n",
    "    \"nonterminal-ontonotes\",\n",
    "    \"constituent-ontonotes\",\n",
    "    \"dep-labeling-ewt\",\n",
    "    \"ner-ontonotes\",\n",
    "    \"srl-conll2012\",\n",
    "    \"coref-ontonotes-conll\",\n",
    "    \"spr1\",\n",
    "    \"spr2\",\n",
    "    \"dpr\",\n",
    "    \"ner-tacred\",\n",
    "    \"rel-tacred\",\n",
    "    \"rel-semeval\",\n",
    "    \"coref-gap\",\n",
    "    \"coref-gap-ontonotes\",\n",
    "    \"noun-verb\",\n",
    "]\n",
    "\n",
    "# See https://bokeh.pydata.org/en/latest/docs/reference/palettes.html\n",
    "_clist = bokeh.palettes.Category20[20]\n",
    "exp_types_clist_idx = [\n",
    "    (\"glove\", 2), # orange\n",
    "    (\"cove\",  6), # deep red\n",
    "    (\"elmo-chars\", 18), # aqua\n",
    "    (\"elmo-ortho\", 8), # purple\n",
    "    (\"elmo-full\", 0), # blue\n",
    "    (\"openai-lex\", 16), # olive\n",
    "    (\"openai-cat\", 4), # green\n",
    "    (\"openai-mix\", 12), # pink\n",
    "    (\"openai\", 4), # green\n",
    "    (\"openai-bwb\", 12), # pink\n",
    "    (\"train-chars\", 10), # brown\n",
    "]\n",
    "# Make all BERT expts the same color\n",
    "for bert_name in ['base-uncased', 'base-cased', 'large-uncased', 'large-cased']:\n",
    "    exp_types_clist_idx.append((f\"bert-{bert_name}-lex\", 16))  # olive\n",
    "    exp_types_clist_idx.append((f\"bert-{bert_name}-cat\", 4))   # green\n",
    "    exp_types_clist_idx.append((f\"bert-{bert_name}-mix\", 12))  # pink\n",
    "    \n",
    "exp_types_colored = collections.OrderedDict()\n",
    "# Use lighter versions for base model, darker for CNN\n",
    "for k, v in exp_types_clist_idx:\n",
    "    exp_types_colored[k] = _clist[v + 1]\n",
    "    exp_types_colored[k+\"-cnn1\"] = _clist[v]\n",
    "    exp_types_colored[k+\"-cnn2\"] = _clist[v]\n",
    "\n",
    "def exp_type_sort_key(candidate):\n",
    "    exp_type = candidate.split(\" \", 1)[0]\n",
    "    return (exp_types.index(exp_type), candidate)\n",
    "\n",
    "def task_sort_key(candidate):\n",
    "    for i, name in enumerate(tasks):\n",
    "        if candidate.startswith(name):\n",
    "            return (i, candidate)\n",
    "    return (len(tasks), candidate)\n",
    "\n",
    "exp_types, palette = zip(*exp_types_colored.items())\n",
    "fill_cmap = bokeh.transform.factor_cmap('exp_type', palette, exp_types)\n",
    "\n",
    "exp_types_colored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_exp_type(exp_name):\n",
    "    m = re.match(r\"([a-z-]+)-edges-([a-z-]+)\", exp_name)\n",
    "    assert m is not None, f\"Unable to parse run name: {run_path}\"\n",
    "    prefix, task = m.groups()\n",
    "    return prefix\n",
    "\n",
    "def clean_task_name(task_name):\n",
    "    c1 = re.sub(r\"^edges-\", \"\", task_name)\n",
    "    c2 = re.sub(r\"-openai$\", \"\", c1)\n",
    "    return c2\n",
    "\n",
    "ID_COLS = ['run', 'task', 'split']\n",
    "\n",
    "def is_core_role(label):\n",
    "    return re.match(r\"^ARG[0-5A]$\", label) is not None\n",
    "\n",
    "def is_non_core_role(label):\n",
    "    return re.match(r\"^ARGM(-.+)?$\", label) is not None\n",
    "\n",
    "def is_core_or_noncore(label):\n",
    "    return is_core_role(label) or is_non_core_role(label)\n",
    "\n",
    "def is_srl_task(task):\n",
    "    return task.startswith(\"srl-\")\n",
    "\n",
    "def is_coref_task(task):\n",
    "    return task.startswith(\"coref-\")\n",
    "\n",
    "def is_constituent_task(task):\n",
    "    return task.startswith(\"constituent-\")\n",
    "\n",
    "def is_relation_task(task):\n",
    "    return task.startswith(\"rel-\")\n",
    "\n",
    "def is_positive_relation(label):\n",
    "    return (not label.startswith(\"_\")) and (label != \"no_relation\") and (label != \"Other\")\n",
    "\n",
    "def agg_label_group(df, task_predicate, label_predicate, group_name):\n",
    "    agg_map = {k:\"sum\" for k in df.columns if k.endswith(\"_count\")}\n",
    "    mask = df['task'].map(task_predicate) & df['label'].map(label_predicate)\n",
    "    sdf = df[mask].groupby(by=ID_COLS).agg(agg_map).reset_index()\n",
    "    sdf['label'] = group_name\n",
    "    return sdf\n",
    "\n",
    "def agg_stratifier_group(df, stratifier, key_predicate, group_name):\n",
    "    agg_map = {k:\"sum\" for k in df.columns if k.endswith(\"_count\")}\n",
    "#     mask = (df['stratifier'] == stratifier) & df['stratum_key'].map(key_predicate)\n",
    "    # Use this for short-circuit evaluation, so we don't call key_predicate on invalid keys\n",
    "    mask = [(s == stratifier and key_predicate(key)) \n",
    "            for s, key in zip(df['stratifier'], df['stratum_key'])]\n",
    "    sdf = df[mask].groupby(by=ID_COLS).agg(agg_map).reset_index()\n",
    "    sdf['label'] = group_name\n",
    "    return sdf    \n",
    "\n",
    "def load_scores_file(filename, tag=None, seed=None):\n",
    "    df = pd.read_csv(filename, sep=\"\\t\", header=0)\n",
    "    df.drop(['Unnamed: 0'], axis='columns', inplace=True)\n",
    "    # df['task_raw'] = df['task'].copy()\n",
    "    df['task'] = df['task'].map(clean_task_name)\n",
    "    if not \"stratifier\" in df.columns:\n",
    "        df[\"stratifier\"] = None\n",
    "    if not \"stratum_key\" in df.columns:\n",
    "        df[\"stratum_key\"] = 0\n",
    "        \n",
    "    ###\n",
    "    # Add additional custom aggregations\n",
    "    _eg = []\n",
    "    # SRL core, non-core, and cleaned micro F1\n",
    "    _eg.append(agg_label_group(df, is_srl_task, is_core_role, \"_core_\"))\n",
    "    _eg.append(agg_label_group(df, is_srl_task, is_non_core_role, \"_non_core_\"))\n",
    "    _eg.append(agg_label_group(df, is_srl_task, is_core_or_noncore, \"_clean_micro_\"))\n",
    "    # Constituents: split into POS, nonterminals\n",
    "    _eg.append(agg_stratifier_group(df, 'info.height', lambda x: int(x) == 1, \"_pos_\"))\n",
    "    _eg.append(agg_stratifier_group(df, 'info.height', lambda x: int(x) > 1, \"_nonterminal_\"))\n",
    "    # Relations: ignore negative class (no_relation)\n",
    "    _eg.append(agg_label_group(df, is_relation_task, is_positive_relation, \"_clean_micro_\"))\n",
    "    df = pd.concat([df] + _eg, ignore_index=True, sort=False)\n",
    "    \n",
    "    df.insert(0, \"exp_name\", df['run'].map(lambda p: os.path.basename(os.path.dirname(p.strip(\"/\")))))\n",
    "    df.insert(1, \"exp_type\", df['exp_name'].map(get_exp_type))\n",
    "    if tag is not None:\n",
    "        df.insert(0, \"tag\", tag)\n",
    "    df.insert(1, \"seed\", seed)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _make_display_name(task, label):\n",
    "    task_to_display_name = {\n",
    "        \"pos-ontonotes\": \"Part-of-Speech\",\n",
    "        \"nonterminal-ontonotes\": \"Constituents\",\n",
    "        \"dep-labeling-ewt\": \"Dependencies\",\n",
    "        \"ner-ontonotes\": \"Entities\",\n",
    "        \"srl-conll2012\": \"SRL\",\n",
    "        \"coref-ontonotes-conll\": \"OntoNotes coref.\",\n",
    "        \"spr1\": \"SPR1\",\n",
    "        \"spr2\": \"SPR2\",\n",
    "        \"dpr\": \"Winograd coref.\",\n",
    "        \"ner-tacred\": \"Entities (TACRED)\",\n",
    "        \"rel-tacred\": \"Relations (TACRED)\",\n",
    "        \"rel-semeval\": \"Relations (SemEval)\",\n",
    "        \"coref-gap\": \"GAP coref.\",\n",
    "        \"coref-gap-ontonotes\": \"GAP coref (train OOD)\",\n",
    "        \"noun-verb\": \"Noun-Verb\",\n",
    "    }\n",
    "    display_task = task_to_display_name[task]\n",
    "    if label in {\"_micro_avg_\", \"1\", None}:\n",
    "        return display_task\n",
    "    elif label == \"_clean_micro_\":\n",
    "        if task.startswith(\"srl-\"):\n",
    "            return f\"{display_task} (all)\"\n",
    "        else:\n",
    "            return display_task\n",
    "    elif label == \"_core_\":\n",
    "        return f\"{display_task} (core)\"\n",
    "    elif label == \"_non_core_\":\n",
    "        return f\"{display_task} (non-core)\"\n",
    "    elif label.endswith(\":_mean_\"):\n",
    "        return f\"{display_task} (mean)\"\n",
    "    else:\n",
    "        clean_label = label.strip(\"_\")\n",
    "        return f\"{display_task} ({clean_label})\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['constituent-ontonotes' 'coref-ontonotes-conll' 'dep-labeling-ewt' 'dpr'\n",
      " 'ner-ontonotes' 'spr1' 'spr2' 'srl-conll2012' 'nonterminal-ontonotes'\n",
      " 'pos-ontonotes' 'rel-tacred' 'rel-semeval' 'noun-verb'\n",
      " 'coref-gap-ontonotes' 'coref-gap' 'ner-tacred']\n",
      "['openai-cat' 'openai-lex' 'openai-bwb' 'cove' 'elmo-chars' 'elmo-full'\n",
      " 'elmo-ortho' 'glove' 'openai' 'bert-base-cased-cat' 'bert-base-cased-lex'\n",
      " 'bert-base-uncased-cat' 'bert-base-uncased-lex' 'bert-large-cased-cat'\n",
      " 'bert-large-cased-lex' 'bert-large-uncased-cat' 'bert-large-uncased-lex'\n",
      " 'bert-base-cased-mix' 'bert-base-uncased-mix' 'bert-large-cased-mix'\n",
      " 'bert-large-uncased-mix' 'openai-mix']\n"
     ]
    }
   ],
   "source": [
    "score_files = []\n",
    "score_files = [\n",
    "    (\"base\", \"/nfs/jsalt/home/iftenney/exp/edges-20180921-openai/scores_cm.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/home/iftenney/exp/edges-20180922-openai/scores_cm.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/home/iftenney/exp/final_20180927/base/scores_cm.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/exp/edges-20190124-bert/scores.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/exp/edges-20190125-bert-splitconst/scores.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/exp/edges-20190125-bert-large/scores.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/exp/edges-20190129-bert-mix/scores.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/exp/edges-20190205-tacred/scores.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/exp/edges-20190205-semeval/scores.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/exp/edges-20190205-noun-verb/scores.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/exp/edges-20190207-gap/scores.tsv\"),\n",
    "    (\"base\", \"/nfs/jsalt/exp/edges-20190209-openai-mix/scores.tsv\"),\n",
    "    (\"bow\", \"/nfs/jsalt/exp/edges-20190220-rel-bow/scores.tsv\"),\n",
    "    (\"bow\", \"/nfs/jsalt/exp/edges-20190220-core-bow/scores.tsv\"),\n",
    "    (\"cnn1\", \"/nfs/jsalt/home/iftenney/exp/final_20180927/cnn1/scores_cm.tsv\"),\n",
    "    (\"cnn2\", \"/nfs/jsalt/home/iftenney/exp/final_20180927/cnn2/scores_cm.tsv\"),\n",
    "]\n",
    "\n",
    "dfs = []\n",
    "for tag, score_file in score_files:\n",
    "    df = load_scores_file(score_file, tag=tag)\n",
    "    dfs.append(df)\n",
    "\n",
    "# Load cross-val score files\n",
    "for seed in [100,101,102,103,104]:\n",
    "    df = load_scores_file(f\"/nfs/jsalt/exp/edges-20190204-dpr-{seed:d}/scores.tsv\", \n",
    "                          tag=\"base\", seed=seed)\n",
    "    dfs.append(df)\n",
    "    df = load_scores_file(f\"/nfs/jsalt/exp/edges-20190213-dpr-openai-mix-{seed:d}/scores.tsv\", \n",
    "                          tag=\"base\", seed=seed)\n",
    "    dfs.append(df)\n",
    "    df = load_scores_file(f\"/nfs/jsalt/exp/edges-20190220-dpr-bert-cat-{seed:d}/scores.tsv\", \n",
    "                          tag=\"base\", seed=seed)\n",
    "    dfs.append(df)\n",
    "\n",
    "df = pd.concat(dfs, ignore_index=True, sort=False)\n",
    "df['display_col'] = [\"%s (%s)\" % et for et in zip(df.exp_type, df.tag)]\n",
    "print(df['task'].unique())\n",
    "print(df['exp_type'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def harmonic_mean(a, b):\n",
    "    return 2 * a * b / (a + b)\n",
    "\n",
    "df['pred_pos_count'] = df.tp_count + df.fp_count\n",
    "df['true_pos_count'] = df.tp_count + df.fn_count\n",
    "df['total_count'] = df.tp_count + df.tn_count + df.fp_count + df.fn_count\n",
    "\n",
    "# NOTE: this overwrites any _macro_avg_ rows by recomputing the micro-average!\n",
    "df['accuracy'] = (df.tp_count + df.tn_count) / df.total_count\n",
    "df['precision'] = df.tp_count / df.pred_pos_count\n",
    "df['recall'] = df.tp_count / df.true_pos_count\n",
    "df['f1_score'] = harmonic_mean(df.precision, df.recall).fillna(0)\n",
    "\n",
    "# Approximate error intervals using normal approximation\n",
    "# https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Wilson_score_interval\n",
    "z = 1.96 # 95% confidence\n",
    "df['accuracy_errn95'] = z * (df.accuracy * (1 - df.accuracy) / df.total_count).map(np.sqrt)\n",
    "df['precision_errn95'] = z * (df.precision * (1 - df.precision) / df.pred_pos_count).map(np.sqrt)\n",
    "df['recall_errn95'] = z * (df.recall * (1 - df.recall) / df.true_pos_count).map(np.sqrt)\n",
    "# This is probably not the right way to combine for F1 score - TODO to figure this out?\n",
    "df['f1_errn95'] = harmonic_mean(df.precision_errn95, df.recall_errn95)\n",
    "\n",
    "def _get_final_score(row):\n",
    "    return row['f1_score'], row['f1_errn95']\n",
    "\n",
    "df['score'], df['score_errn95'] = zip(*(_get_final_score(row) for i, row in df.iterrows()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For DPR, we need to average across multiple runs to get a good estimate of performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = df['task'] == 'dpr'\n",
    "mask &= df['label'] != \"__run_info__\"\n",
    "mask &= df['seed'].notnull()\n",
    "gb_cols = [\"tag\", \"exp_name\", \"exp_type\", \"task\", \"label\", \"split\", \"display_col\"]\n",
    "gb = df[mask].groupby(by=gb_cols)\n",
    "new_rows = []\n",
    "for key, idxs in gb.groups.items():\n",
    "    new_row = dict(zip(gb_cols, key))\n",
    "    new_row[\"seed\"] = \"_mean_\"\n",
    "    new_row[\"score\"] = df.loc[idxs, \"score\"].mean()\n",
    "    new_row[\"score_errn95\"] = 1.96 * np.sqrt(df.loc[idxs, \"score\"].var()/len(idxs))\n",
    "    new_rows.append(new_row)\n",
    "    \n",
    "agg_df = pd.DataFrame.from_records(new_rows)\n",
    "df = pd.concat([df, agg_df], ignore_index=True, sort=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For SemEval 2010 Task 8, the official metric is macro-averaged F1 over non-Other labels. Compute this so we can compare to SOTA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "run,split,score\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-base-uncased-cat-edges-rel-semeval/run,test,0.7033\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-base-uncased-cat-edges-rel-semeval/run,val,0.7056\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-base-uncased-lex-edges-rel-semeval/run,test,0.5326\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-base-uncased-lex-edges-rel-semeval/run,val,0.5021\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-base-uncased-mix-edges-rel-semeval/run,test,0.7441\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-base-uncased-mix-edges-rel-semeval/run,val,0.7556\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-large-uncased-cat-edges-rel-semeval/run,test,0.7041\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-large-uncased-cat-edges-rel-semeval/run,val,0.6980\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-large-uncased-lex-edges-rel-semeval/run,test,0.5254\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-large-uncased-lex-edges-rel-semeval/run,val,0.5309\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-large-uncased-mix-edges-rel-semeval/run,test,0.7558\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/bert-large-uncased-mix-edges-rel-semeval/run,val,0.7583\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/cove-edges-rel-semeval/run,test,0.5385\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/cove-edges-rel-semeval/run,val,0.6719\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/elmo-chars-edges-rel-semeval/run,test,0.5096\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/elmo-chars-edges-rel-semeval/run,val,0.4717\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/elmo-full-edges-rel-semeval/run,test,0.7006\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/elmo-full-edges-rel-semeval/run,val,0.6831\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/elmo-ortho-edges-rel-semeval/run_seed_0,test,0.5143\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/elmo-ortho-edges-rel-semeval/run_seed_0,val,0.4902\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/glove-edges-rel-semeval/run,test,0.4474\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/glove-edges-rel-semeval/run,val,0.4846\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/openai-cat-edges-rel-semeval/run,test,0.7330\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/openai-cat-edges-rel-semeval/run,val,0.7471\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/openai-lex-edges-rel-semeval/run,test,0.5348\n",
      "/nfs/jsalt/exp/edges-20190205-semeval/openai-lex-edges-rel-semeval/run,val,0.5258\n",
      "/nfs/jsalt/exp/edges-20190209-openai-mix/openai-mix-edges-rel-semeval/run,test,0.7214\n",
      "/nfs/jsalt/exp/edges-20190209-openai-mix/openai-mix-edges-rel-semeval/run,val,0.7444\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//bert-base-uncased-lex-edges-rel-semeval/run,test,0.6458\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//bert-base-uncased-lex-edges-rel-semeval/run,val,0.6378\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//bert-large-uncased-lex-edges-rel-semeval/run,test,0.6323\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//bert-large-uncased-lex-edges-rel-semeval/run,val,0.6236\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//elmo-chars-edges-rel-semeval/run,test,0.6478\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//elmo-chars-edges-rel-semeval/run,val,0.6178\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//glove-edges-rel-semeval/run,test,0.5604\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//glove-edges-rel-semeval/run,val,0.6325\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//openai-lex-edges-rel-semeval/run,test,0.6279\n",
      "/nfs/jsalt/exp/edges-20190220-rel-bow//openai-lex-edges-rel-semeval/run,val,0.6414\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mask = df['task'] == 'rel-semeval'\n",
    "mask &= df['split'].notnull()\n",
    "mask &= df['label'].map(is_positive_relation)\n",
    "_id_cols = ['run', 'split']\n",
    "_agg_cols = ['score']\n",
    "gb = df[mask][_id_cols + _agg_cols].groupby(_id_cols)\n",
    "afd = gb.agg('mean')\n",
    "afd = afd.reset_index()\n",
    "\n",
    "csv_args = dict(float_format=\"%.4f\")\n",
    "print(afd.to_csv(index=False, **csv_args))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute clean metrics for each task\n",
    "\n",
    "For most tasks this is just the micro or macro average F1, but we need to ignore the 0 label for coref, and drop references and continuations for SRL."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "269\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>seed</th>\n",
       "      <th>exp_name</th>\n",
       "      <th>exp_type</th>\n",
       "      <th>label</th>\n",
       "      <th>num_epochs</th>\n",
       "      <th>num_steps</th>\n",
       "      <th>run</th>\n",
       "      <th>task</th>\n",
       "      <th>split</th>\n",
       "      <th>...</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>accuracy_errn95</th>\n",
       "      <th>precision_errn95</th>\n",
       "      <th>recall_errn95</th>\n",
       "      <th>f1_errn95</th>\n",
       "      <th>score</th>\n",
       "      <th>score_errn95</th>\n",
       "      <th>display_row</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-cat-edges-coref-ontonotes-conll</td>\n",
       "      <td>openai-cat</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/edges-20180921-op...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.855263</td>\n",
       "      <td>0.817258</td>\n",
       "      <td>0.835829</td>\n",
       "      <td>0.003098</td>\n",
       "      <td>0.009258</td>\n",
       "      <td>0.009941</td>\n",
       "      <td>0.009587</td>\n",
       "      <td>0.835829</td>\n",
       "      <td>0.009587</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-lex-edges-coref-ontonotes-conll</td>\n",
       "      <td>openai-lex</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/edges-20180921-op...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.808771</td>\n",
       "      <td>0.660696</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.003768</td>\n",
       "      <td>0.011192</td>\n",
       "      <td>0.012179</td>\n",
       "      <td>0.011665</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.011665</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-bwb-edges-coref-ontonotes-conll</td>\n",
       "      <td>openai-bwb</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/edges-20180922-op...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.855844</td>\n",
       "      <td>0.810885</td>\n",
       "      <td>0.832758</td>\n",
       "      <td>0.003118</td>\n",
       "      <td>0.009282</td>\n",
       "      <td>0.010073</td>\n",
       "      <td>0.009661</td>\n",
       "      <td>0.832758</td>\n",
       "      <td>0.009661</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>cove-edges-coref-ontonotes-conll</td>\n",
       "      <td>cove</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.835032</td>\n",
       "      <td>0.761970</td>\n",
       "      <td>0.796830</td>\n",
       "      <td>0.003380</td>\n",
       "      <td>0.009994</td>\n",
       "      <td>0.010955</td>\n",
       "      <td>0.010452</td>\n",
       "      <td>0.796830</td>\n",
       "      <td>0.010452</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-chars-edges-coref-ontonotes-conll</td>\n",
       "      <td>elmo-chars</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.802593</td>\n",
       "      <td>0.714261</td>\n",
       "      <td>0.755855</td>\n",
       "      <td>0.003651</td>\n",
       "      <td>0.010853</td>\n",
       "      <td>0.011621</td>\n",
       "      <td>0.011224</td>\n",
       "      <td>0.755855</td>\n",
       "      <td>0.011224</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-full-edges-coref-ontonotes-conll</td>\n",
       "      <td>elmo-full</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.861156</td>\n",
       "      <td>0.834309</td>\n",
       "      <td>0.847520</td>\n",
       "      <td>0.003003</td>\n",
       "      <td>0.009036</td>\n",
       "      <td>0.009564</td>\n",
       "      <td>0.009293</td>\n",
       "      <td>0.847520</td>\n",
       "      <td>0.009293</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-ortho-edges-coref-ontonotes-conll</td>\n",
       "      <td>elmo-ortho</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.812175</td>\n",
       "      <td>0.781261</td>\n",
       "      <td>0.796418</td>\n",
       "      <td>0.003423</td>\n",
       "      <td>0.010243</td>\n",
       "      <td>0.010634</td>\n",
       "      <td>0.010435</td>\n",
       "      <td>0.796418</td>\n",
       "      <td>0.010435</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>glove-edges-coref-ontonotes-conll</td>\n",
       "      <td>glove</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.793172</td>\n",
       "      <td>0.684292</td>\n",
       "      <td>0.734720</td>\n",
       "      <td>0.003763</td>\n",
       "      <td>0.011217</td>\n",
       "      <td>0.011956</td>\n",
       "      <td>0.011575</td>\n",
       "      <td>0.734720</td>\n",
       "      <td>0.011575</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-base-uncased-cat-edges-coref-ontonotes-conll</td>\n",
       "      <td>bert-base-uncased-cat</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190124-bert/bert-base-u...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.892236</td>\n",
       "      <td>0.892697</td>\n",
       "      <td>0.892467</td>\n",
       "      <td>0.002567</td>\n",
       "      <td>0.007974</td>\n",
       "      <td>0.007961</td>\n",
       "      <td>0.007968</td>\n",
       "      <td>0.892467</td>\n",
       "      <td>0.007968</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-base-uncased-lex-edges-coref-ontonotes-conll</td>\n",
       "      <td>bert-base-uncased-lex</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190124-bert/bert-base-u...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.797980</td>\n",
       "      <td>0.721151</td>\n",
       "      <td>0.757622</td>\n",
       "      <td>0.003651</td>\n",
       "      <td>0.010864</td>\n",
       "      <td>0.011535</td>\n",
       "      <td>0.011189</td>\n",
       "      <td>0.757622</td>\n",
       "      <td>0.011189</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-large-uncased-cat-edges-coref-ontonotes-c...</td>\n",
       "      <td>bert-large-uncased-cat</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190125-bert-large/bert-...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.894575</td>\n",
       "      <td>0.903204</td>\n",
       "      <td>0.898869</td>\n",
       "      <td>0.002499</td>\n",
       "      <td>0.007862</td>\n",
       "      <td>0.007606</td>\n",
       "      <td>0.007732</td>\n",
       "      <td>0.898869</td>\n",
       "      <td>0.007732</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-large-uncased-lex-edges-coref-ontonotes-c...</td>\n",
       "      <td>bert-large-uncased-lex</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190125-bert-large/bert-...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.803609</td>\n",
       "      <td>0.720978</td>\n",
       "      <td>0.760054</td>\n",
       "      <td>0.003629</td>\n",
       "      <td>0.010789</td>\n",
       "      <td>0.011537</td>\n",
       "      <td>0.011150</td>\n",
       "      <td>0.760054</td>\n",
       "      <td>0.011150</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-base-uncased-mix-edges-coref-ontonotes-conll</td>\n",
       "      <td>bert-base-uncased-mix</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190129-bert-mix/bert-ba...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.905996</td>\n",
       "      <td>0.903031</td>\n",
       "      <td>0.904511</td>\n",
       "      <td>0.002424</td>\n",
       "      <td>0.007519</td>\n",
       "      <td>0.007612</td>\n",
       "      <td>0.007565</td>\n",
       "      <td>0.904511</td>\n",
       "      <td>0.007565</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-large-uncased-mix-edges-coref-ontonotes-c...</td>\n",
       "      <td>bert-large-uncased-mix</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190129-bert-mix/bert-la...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.920096</td>\n",
       "      <td>0.924216</td>\n",
       "      <td>0.922152</td>\n",
       "      <td>0.002201</td>\n",
       "      <td>0.006959</td>\n",
       "      <td>0.006808</td>\n",
       "      <td>0.006882</td>\n",
       "      <td>0.922152</td>\n",
       "      <td>0.006882</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-mix-edges-coref-ontonotes-conll</td>\n",
       "      <td>openai-mix</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190209-openai-mix/opena...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.861145</td>\n",
       "      <td>0.863073</td>\n",
       "      <td>0.862108</td>\n",
       "      <td>0.002888</td>\n",
       "      <td>0.008885</td>\n",
       "      <td>0.008843</td>\n",
       "      <td>0.008864</td>\n",
       "      <td>0.862108</td>\n",
       "      <td>0.008864</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-base-uncased-lex-edges-coref-ontonotes-conll</td>\n",
       "      <td>bert-base-uncased-lex</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-core-bow/bert-ba...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.802081</td>\n",
       "      <td>0.756631</td>\n",
       "      <td>0.778694</td>\n",
       "      <td>0.003539</td>\n",
       "      <td>0.010552</td>\n",
       "      <td>0.011038</td>\n",
       "      <td>0.010790</td>\n",
       "      <td>0.778694</td>\n",
       "      <td>0.010790</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-large-uncased-lex-edges-coref-ontonotes-c...</td>\n",
       "      <td>bert-large-uncased-lex</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-core-bow/bert-la...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.804084</td>\n",
       "      <td>0.752842</td>\n",
       "      <td>0.777620</td>\n",
       "      <td>0.003541</td>\n",
       "      <td>0.010551</td>\n",
       "      <td>0.011096</td>\n",
       "      <td>0.010817</td>\n",
       "      <td>0.777620</td>\n",
       "      <td>0.010817</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-chars-edges-coref-ontonotes-conll</td>\n",
       "      <td>elmo-chars</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-core-bow/elmo-ch...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.808431</td>\n",
       "      <td>0.746469</td>\n",
       "      <td>0.776216</td>\n",
       "      <td>0.003540</td>\n",
       "      <td>0.010535</td>\n",
       "      <td>0.011190</td>\n",
       "      <td>0.010853</td>\n",
       "      <td>0.776216</td>\n",
       "      <td>0.010853</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>glove-edges-coref-ontonotes-conll</td>\n",
       "      <td>glove</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-core-bow/glove-e...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.808152</td>\n",
       "      <td>0.734240</td>\n",
       "      <td>0.769425</td>\n",
       "      <td>0.003575</td>\n",
       "      <td>0.010626</td>\n",
       "      <td>0.011363</td>\n",
       "      <td>0.010982</td>\n",
       "      <td>0.769425</td>\n",
       "      <td>0.010982</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-lex-edges-coref-ontonotes-conll</td>\n",
       "      <td>openai-lex</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-core-bow/openai-...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.820021</td>\n",
       "      <td>0.670169</td>\n",
       "      <td>0.737560</td>\n",
       "      <td>0.003705</td>\n",
       "      <td>0.010931</td>\n",
       "      <td>0.012094</td>\n",
       "      <td>0.011483</td>\n",
       "      <td>0.737560</td>\n",
       "      <td>0.011483</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>cnn1</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-chars-edges-coref-ontonotes-conll</td>\n",
       "      <td>elmo-chars</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/cn...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.824132</td>\n",
       "      <td>0.776438</td>\n",
       "      <td>0.799574</td>\n",
       "      <td>0.003383</td>\n",
       "      <td>0.010089</td>\n",
       "      <td>0.010717</td>\n",
       "      <td>0.010394</td>\n",
       "      <td>0.799574</td>\n",
       "      <td>0.010394</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>cnn2</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-chars-edges-coref-ontonotes-conll</td>\n",
       "      <td>elmo-chars</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/cn...</td>\n",
       "      <td>coref-ontonotes-conll</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.826167</td>\n",
       "      <td>0.774371</td>\n",
       "      <td>0.799431</td>\n",
       "      <td>0.003380</td>\n",
       "      <td>0.010069</td>\n",
       "      <td>0.010752</td>\n",
       "      <td>0.010399</td>\n",
       "      <td>0.799431</td>\n",
       "      <td>0.010399</td>\n",
       "      <td>coref-ontonotes-conll-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-cat-edges-dep-labeling-ewt</td>\n",
       "      <td>openai-cat</td>\n",
       "      <td>_micro_avg_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/edges-20180921-op...</td>\n",
       "      <td>dep-labeling-ewt</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.937647</td>\n",
       "      <td>0.904301</td>\n",
       "      <td>0.920672</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>0.003045</td>\n",
       "      <td>0.003639</td>\n",
       "      <td>0.003316</td>\n",
       "      <td>0.920672</td>\n",
       "      <td>0.003316</td>\n",
       "      <td>dep-labeling-ewt-_micro_avg_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-lex-edges-dep-labeling-ewt</td>\n",
       "      <td>openai-lex</td>\n",
       "      <td>_micro_avg_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/edges-20180921-op...</td>\n",
       "      <td>dep-labeling-ewt</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.887963</td>\n",
       "      <td>0.691557</td>\n",
       "      <td>0.777549</td>\n",
       "      <td>0.000155</td>\n",
       "      <td>0.004421</td>\n",
       "      <td>0.005713</td>\n",
       "      <td>0.004984</td>\n",
       "      <td>0.777549</td>\n",
       "      <td>0.004984</td>\n",
       "      <td>dep-labeling-ewt-_micro_avg_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-bwb-edges-dep-labeling-ewt</td>\n",
       "      <td>openai-bwb</td>\n",
       "      <td>_micro_avg_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/edges-20180922-op...</td>\n",
       "      <td>dep-labeling-ewt</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.938856</td>\n",
       "      <td>0.905018</td>\n",
       "      <td>0.921626</td>\n",
       "      <td>0.000097</td>\n",
       "      <td>0.003018</td>\n",
       "      <td>0.003626</td>\n",
       "      <td>0.003295</td>\n",
       "      <td>0.921626</td>\n",
       "      <td>0.003295</td>\n",
       "      <td>dep-labeling-ewt-_micro_avg_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>cove-edges-dep-labeling-ewt</td>\n",
       "      <td>cove</td>\n",
       "      <td>_micro_avg_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>dep-labeling-ewt</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.924871</td>\n",
       "      <td>0.878057</td>\n",
       "      <td>0.900856</td>\n",
       "      <td>0.000109</td>\n",
       "      <td>0.003346</td>\n",
       "      <td>0.004047</td>\n",
       "      <td>0.003664</td>\n",
       "      <td>0.900856</td>\n",
       "      <td>0.003664</td>\n",
       "      <td>dep-labeling-ewt-_micro_avg_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-chars-edges-dep-labeling-ewt</td>\n",
       "      <td>elmo-chars</td>\n",
       "      <td>_micro_avg_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>dep-labeling-ewt</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.898903</td>\n",
       "      <td>0.730864</td>\n",
       "      <td>0.806221</td>\n",
       "      <td>0.000146</td>\n",
       "      <td>0.004135</td>\n",
       "      <td>0.005486</td>\n",
       "      <td>0.004716</td>\n",
       "      <td>0.806221</td>\n",
       "      <td>0.004716</td>\n",
       "      <td>dep-labeling-ewt-_micro_avg_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-full-edges-dep-labeling-ewt</td>\n",
       "      <td>elmo-full</td>\n",
       "      <td>_micro_avg_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>dep-labeling-ewt</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.957174</td>\n",
       "      <td>0.927479</td>\n",
       "      <td>0.942093</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.002544</td>\n",
       "      <td>0.003208</td>\n",
       "      <td>0.002838</td>\n",
       "      <td>0.942093</td>\n",
       "      <td>0.002838</td>\n",
       "      <td>dep-labeling-ewt-_micro_avg_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-ortho-edges-dep-labeling-ewt</td>\n",
       "      <td>elmo-ortho</td>\n",
       "      <td>_micro_avg_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>dep-labeling-ewt</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.912583</td>\n",
       "      <td>0.800319</td>\n",
       "      <td>0.852772</td>\n",
       "      <td>0.000130</td>\n",
       "      <td>0.003731</td>\n",
       "      <td>0.004945</td>\n",
       "      <td>0.004253</td>\n",
       "      <td>0.852772</td>\n",
       "      <td>0.004253</td>\n",
       "      <td>dep-labeling-ewt-_micro_avg_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>glove-edges-dep-labeling-ewt</td>\n",
       "      <td>glove</td>\n",
       "      <td>_micro_avg_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/home/iftenney/exp/final_20180927/ba...</td>\n",
       "      <td>dep-labeling-ewt</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.899781</td>\n",
       "      <td>0.671485</td>\n",
       "      <td>0.769048</td>\n",
       "      <td>0.000156</td>\n",
       "      <td>0.004300</td>\n",
       "      <td>0.005809</td>\n",
       "      <td>0.004942</td>\n",
       "      <td>0.769048</td>\n",
       "      <td>0.004942</td>\n",
       "      <td>dep-labeling-ewt-_micro_avg_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-full-edges-rel-tacred</td>\n",
       "      <td>elmo-full</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-tacred/elmo-full...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.705402</td>\n",
       "      <td>0.413171</td>\n",
       "      <td>0.521114</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>0.015834</td>\n",
       "      <td>0.013090</td>\n",
       "      <td>0.014332</td>\n",
       "      <td>0.521114</td>\n",
       "      <td>0.014332</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-ortho-edges-rel-tacred</td>\n",
       "      <td>elmo-ortho</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-tacred/elmo-orth...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.614417</td>\n",
       "      <td>0.233628</td>\n",
       "      <td>0.338531</td>\n",
       "      <td>0.000148</td>\n",
       "      <td>0.020983</td>\n",
       "      <td>0.011249</td>\n",
       "      <td>0.014646</td>\n",
       "      <td>0.338531</td>\n",
       "      <td>0.014646</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>glove-edges-rel-tacred</td>\n",
       "      <td>glove</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-tacred/glove-edg...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.612549</td>\n",
       "      <td>0.174209</td>\n",
       "      <td>0.271269</td>\n",
       "      <td>0.000150</td>\n",
       "      <td>0.024285</td>\n",
       "      <td>0.010083</td>\n",
       "      <td>0.014249</td>\n",
       "      <td>0.271269</td>\n",
       "      <td>0.014249</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-cat-edges-rel-tacred</td>\n",
       "      <td>openai-cat</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-tacred/openai-ca...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.707666</td>\n",
       "      <td>0.309051</td>\n",
       "      <td>0.430218</td>\n",
       "      <td>0.000141</td>\n",
       "      <td>0.018297</td>\n",
       "      <td>0.012284</td>\n",
       "      <td>0.014699</td>\n",
       "      <td>0.430218</td>\n",
       "      <td>0.014699</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-lex-edges-rel-tacred</td>\n",
       "      <td>openai-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-tacred/openai-le...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.579454</td>\n",
       "      <td>0.132818</td>\n",
       "      <td>0.216103</td>\n",
       "      <td>0.000152</td>\n",
       "      <td>0.027410</td>\n",
       "      <td>0.009022</td>\n",
       "      <td>0.013576</td>\n",
       "      <td>0.216103</td>\n",
       "      <td>0.013576</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-mix-edges-rel-tacred</td>\n",
       "      <td>openai-mix</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190209-openai-mix/opena...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.682254</td>\n",
       "      <td>0.327447</td>\n",
       "      <td>0.442511</td>\n",
       "      <td>0.000141</td>\n",
       "      <td>0.017866</td>\n",
       "      <td>0.012475</td>\n",
       "      <td>0.014692</td>\n",
       "      <td>0.442511</td>\n",
       "      <td>0.014692</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-base-uncased-lex-edges-rel-tacred</td>\n",
       "      <td>bert-base-uncased-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//bert-ba...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.646013</td>\n",
       "      <td>0.247425</td>\n",
       "      <td>0.357808</td>\n",
       "      <td>0.000146</td>\n",
       "      <td>0.020541</td>\n",
       "      <td>0.011471</td>\n",
       "      <td>0.014721</td>\n",
       "      <td>0.357808</td>\n",
       "      <td>0.014721</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-large-uncased-lex-edges-rel-tacred</td>\n",
       "      <td>bert-large-uncased-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//bert-la...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.650498</td>\n",
       "      <td>0.180096</td>\n",
       "      <td>0.282092</td>\n",
       "      <td>0.000149</td>\n",
       "      <td>0.024090</td>\n",
       "      <td>0.010215</td>\n",
       "      <td>0.014347</td>\n",
       "      <td>0.282092</td>\n",
       "      <td>0.014347</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-chars-edges-rel-tacred</td>\n",
       "      <td>elmo-chars</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//elmo-ch...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.639163</td>\n",
       "      <td>0.286424</td>\n",
       "      <td>0.395579</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.019071</td>\n",
       "      <td>0.012018</td>\n",
       "      <td>0.014745</td>\n",
       "      <td>0.395579</td>\n",
       "      <td>0.014745</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>glove-edges-rel-tacred</td>\n",
       "      <td>glove</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//glove-e...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.635789</td>\n",
       "      <td>0.258830</td>\n",
       "      <td>0.367891</td>\n",
       "      <td>0.000146</td>\n",
       "      <td>0.020049</td>\n",
       "      <td>0.011643</td>\n",
       "      <td>0.014732</td>\n",
       "      <td>0.367891</td>\n",
       "      <td>0.014732</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-lex-edges-rel-tacred</td>\n",
       "      <td>openai-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//openai-...</td>\n",
       "      <td>rel-tacred</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.617951</td>\n",
       "      <td>0.241906</td>\n",
       "      <td>0.347700</td>\n",
       "      <td>0.000148</td>\n",
       "      <td>0.020645</td>\n",
       "      <td>0.011384</td>\n",
       "      <td>0.014676</td>\n",
       "      <td>0.347700</td>\n",
       "      <td>0.014676</td>\n",
       "      <td>rel-tacred-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-base-uncased-cat-edges-rel-semeval</td>\n",
       "      <td>bert-base-uncased-cat</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/bert-bas...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.891473</td>\n",
       "      <td>0.718002</td>\n",
       "      <td>0.795389</td>\n",
       "      <td>0.001770</td>\n",
       "      <td>0.021913</td>\n",
       "      <td>0.028450</td>\n",
       "      <td>0.024757</td>\n",
       "      <td>0.795389</td>\n",
       "      <td>0.024757</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-base-uncased-lex-edges-rel-semeval</td>\n",
       "      <td>bert-base-uncased-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/bert-bas...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.754653</td>\n",
       "      <td>0.464100</td>\n",
       "      <td>0.574742</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.034692</td>\n",
       "      <td>0.031531</td>\n",
       "      <td>0.033036</td>\n",
       "      <td>0.574742</td>\n",
       "      <td>0.033036</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-base-uncased-mix-edges-rel-semeval</td>\n",
       "      <td>bert-base-uncased-mix</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/bert-bas...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.891667</td>\n",
       "      <td>0.779396</td>\n",
       "      <td>0.831760</td>\n",
       "      <td>0.001637</td>\n",
       "      <td>0.021018</td>\n",
       "      <td>0.026217</td>\n",
       "      <td>0.023332</td>\n",
       "      <td>0.831760</td>\n",
       "      <td>0.023332</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-large-uncased-cat-edges-rel-semeval</td>\n",
       "      <td>bert-large-uncased-cat</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/bert-lar...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.876448</td>\n",
       "      <td>0.708637</td>\n",
       "      <td>0.783659</td>\n",
       "      <td>0.001821</td>\n",
       "      <td>0.023138</td>\n",
       "      <td>0.028729</td>\n",
       "      <td>0.025632</td>\n",
       "      <td>0.783659</td>\n",
       "      <td>0.025632</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-large-uncased-lex-edges-rel-semeval</td>\n",
       "      <td>bert-large-uncased-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/bert-lar...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.757774</td>\n",
       "      <td>0.481790</td>\n",
       "      <td>0.589059</td>\n",
       "      <td>0.002371</td>\n",
       "      <td>0.033972</td>\n",
       "      <td>0.031592</td>\n",
       "      <td>0.032739</td>\n",
       "      <td>0.589059</td>\n",
       "      <td>0.032739</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-large-uncased-mix-edges-rel-semeval</td>\n",
       "      <td>bert-large-uncased-mix</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/bert-lar...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.894366</td>\n",
       "      <td>0.792924</td>\n",
       "      <td>0.840596</td>\n",
       "      <td>0.001600</td>\n",
       "      <td>0.020639</td>\n",
       "      <td>0.025620</td>\n",
       "      <td>0.022861</td>\n",
       "      <td>0.840596</td>\n",
       "      <td>0.022861</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>cove-edges-rel-semeval</td>\n",
       "      <td>cove</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/cove-edg...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.871693</td>\n",
       "      <td>0.685744</td>\n",
       "      <td>0.767618</td>\n",
       "      <td>0.001875</td>\n",
       "      <td>0.023840</td>\n",
       "      <td>0.029351</td>\n",
       "      <td>0.026310</td>\n",
       "      <td>0.767618</td>\n",
       "      <td>0.026310</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-chars-edges-rel-semeval</td>\n",
       "      <td>elmo-chars</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/elmo-cha...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.704274</td>\n",
       "      <td>0.428720</td>\n",
       "      <td>0.532988</td>\n",
       "      <td>0.002502</td>\n",
       "      <td>0.036982</td>\n",
       "      <td>0.031290</td>\n",
       "      <td>0.033899</td>\n",
       "      <td>0.532988</td>\n",
       "      <td>0.033899</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-full-edges-rel-semeval</td>\n",
       "      <td>elmo-full</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/elmo-ful...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.869001</td>\n",
       "      <td>0.697190</td>\n",
       "      <td>0.773672</td>\n",
       "      <td>0.001858</td>\n",
       "      <td>0.023816</td>\n",
       "      <td>0.029051</td>\n",
       "      <td>0.026174</td>\n",
       "      <td>0.773672</td>\n",
       "      <td>0.026174</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-ortho-edges-rel-semeval</td>\n",
       "      <td>elmo-ortho</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/elmo-ort...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.744863</td>\n",
       "      <td>0.452653</td>\n",
       "      <td>0.563107</td>\n",
       "      <td>0.002422</td>\n",
       "      <td>0.035357</td>\n",
       "      <td>0.031471</td>\n",
       "      <td>0.033301</td>\n",
       "      <td>0.563107</td>\n",
       "      <td>0.033301</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>glove-edges-rel-semeval</td>\n",
       "      <td>glove</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/glove-ed...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.780847</td>\n",
       "      <td>0.441207</td>\n",
       "      <td>0.563830</td>\n",
       "      <td>0.002388</td>\n",
       "      <td>0.034795</td>\n",
       "      <td>0.031394</td>\n",
       "      <td>0.033007</td>\n",
       "      <td>0.563830</td>\n",
       "      <td>0.033007</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-cat-edges-rel-semeval</td>\n",
       "      <td>openai-cat</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/openai-c...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.871043</td>\n",
       "      <td>0.773153</td>\n",
       "      <td>0.819184</td>\n",
       "      <td>0.001703</td>\n",
       "      <td>0.022492</td>\n",
       "      <td>0.026479</td>\n",
       "      <td>0.024323</td>\n",
       "      <td>0.819184</td>\n",
       "      <td>0.024323</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-lex-edges-rel-semeval</td>\n",
       "      <td>openai-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190205-semeval/openai-l...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.765000</td>\n",
       "      <td>0.477627</td>\n",
       "      <td>0.588085</td>\n",
       "      <td>0.002365</td>\n",
       "      <td>0.033927</td>\n",
       "      <td>0.031581</td>\n",
       "      <td>0.032712</td>\n",
       "      <td>0.588085</td>\n",
       "      <td>0.032712</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>base</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-mix-edges-rel-semeval</td>\n",
       "      <td>openai-mix</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190209-openai-mix/opena...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.855056</td>\n",
       "      <td>0.791883</td>\n",
       "      <td>0.822258</td>\n",
       "      <td>0.001705</td>\n",
       "      <td>0.023129</td>\n",
       "      <td>0.025667</td>\n",
       "      <td>0.024332</td>\n",
       "      <td>0.822258</td>\n",
       "      <td>0.024332</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-base-uncased-lex-edges-rel-semeval</td>\n",
       "      <td>bert-base-uncased-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//bert-ba...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.813738</td>\n",
       "      <td>0.640999</td>\n",
       "      <td>0.717113</td>\n",
       "      <td>0.002065</td>\n",
       "      <td>0.027734</td>\n",
       "      <td>0.030330</td>\n",
       "      <td>0.028974</td>\n",
       "      <td>0.717113</td>\n",
       "      <td>0.028974</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>bert-large-uncased-lex-edges-rel-semeval</td>\n",
       "      <td>bert-large-uncased-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//bert-la...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.786260</td>\n",
       "      <td>0.643080</td>\n",
       "      <td>0.707499</td>\n",
       "      <td>0.002116</td>\n",
       "      <td>0.028660</td>\n",
       "      <td>0.030291</td>\n",
       "      <td>0.029453</td>\n",
       "      <td>0.707499</td>\n",
       "      <td>0.029453</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>elmo-chars-edges-rel-semeval</td>\n",
       "      <td>elmo-chars</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//elmo-ch...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.802372</td>\n",
       "      <td>0.633715</td>\n",
       "      <td>0.708140</td>\n",
       "      <td>0.002097</td>\n",
       "      <td>0.028330</td>\n",
       "      <td>0.030461</td>\n",
       "      <td>0.029357</td>\n",
       "      <td>0.708140</td>\n",
       "      <td>0.029357</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>glove-edges-rel-semeval</td>\n",
       "      <td>glove</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//glove-e...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.806242</td>\n",
       "      <td>0.645161</td>\n",
       "      <td>0.716763</td>\n",
       "      <td>0.002073</td>\n",
       "      <td>0.027935</td>\n",
       "      <td>0.030251</td>\n",
       "      <td>0.029047</td>\n",
       "      <td>0.716763</td>\n",
       "      <td>0.029047</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>bow</td>\n",
       "      <td>None</td>\n",
       "      <td>openai-lex-edges-rel-semeval</td>\n",
       "      <td>openai-lex</td>\n",
       "      <td>_clean_micro_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/nfs/jsalt/exp/edges-20190220-rel-bow//openai-...</td>\n",
       "      <td>rel-semeval</td>\n",
       "      <td>val</td>\n",
       "      <td>...</td>\n",
       "      <td>0.825166</td>\n",
       "      <td>0.648283</td>\n",
       "      <td>0.726107</td>\n",
       "      <td>0.002031</td>\n",
       "      <td>0.027094</td>\n",
       "      <td>0.030191</td>\n",
       "      <td>0.028558</td>\n",
       "      <td>0.726107</td>\n",
       "      <td>0.028558</td>\n",
       "      <td>rel-semeval-_clean_micro_</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>269 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      tag  seed                                           exp_name  \\\n",
       "0    base  None             openai-cat-edges-coref-ontonotes-conll   \n",
       "1    base  None             openai-lex-edges-coref-ontonotes-conll   \n",
       "2    base  None             openai-bwb-edges-coref-ontonotes-conll   \n",
       "3    base  None                   cove-edges-coref-ontonotes-conll   \n",
       "4    base  None             elmo-chars-edges-coref-ontonotes-conll   \n",
       "5    base  None              elmo-full-edges-coref-ontonotes-conll   \n",
       "6    base  None             elmo-ortho-edges-coref-ontonotes-conll   \n",
       "7    base  None                  glove-edges-coref-ontonotes-conll   \n",
       "8    base  None  bert-base-uncased-cat-edges-coref-ontonotes-conll   \n",
       "9    base  None  bert-base-uncased-lex-edges-coref-ontonotes-conll   \n",
       "10   base  None  bert-large-uncased-cat-edges-coref-ontonotes-c...   \n",
       "11   base  None  bert-large-uncased-lex-edges-coref-ontonotes-c...   \n",
       "12   base  None  bert-base-uncased-mix-edges-coref-ontonotes-conll   \n",
       "13   base  None  bert-large-uncased-mix-edges-coref-ontonotes-c...   \n",
       "14   base  None             openai-mix-edges-coref-ontonotes-conll   \n",
       "15    bow  None  bert-base-uncased-lex-edges-coref-ontonotes-conll   \n",
       "16    bow  None  bert-large-uncased-lex-edges-coref-ontonotes-c...   \n",
       "17    bow  None             elmo-chars-edges-coref-ontonotes-conll   \n",
       "18    bow  None                  glove-edges-coref-ontonotes-conll   \n",
       "19    bow  None             openai-lex-edges-coref-ontonotes-conll   \n",
       "20   cnn1  None             elmo-chars-edges-coref-ontonotes-conll   \n",
       "21   cnn2  None             elmo-chars-edges-coref-ontonotes-conll   \n",
       "22   base  None                  openai-cat-edges-dep-labeling-ewt   \n",
       "23   base  None                  openai-lex-edges-dep-labeling-ewt   \n",
       "24   base  None                  openai-bwb-edges-dep-labeling-ewt   \n",
       "25   base  None                        cove-edges-dep-labeling-ewt   \n",
       "26   base  None                  elmo-chars-edges-dep-labeling-ewt   \n",
       "27   base  None                   elmo-full-edges-dep-labeling-ewt   \n",
       "28   base  None                  elmo-ortho-edges-dep-labeling-ewt   \n",
       "29   base  None                       glove-edges-dep-labeling-ewt   \n",
       "..    ...   ...                                                ...   \n",
       "239  base  None                         elmo-full-edges-rel-tacred   \n",
       "240  base  None                        elmo-ortho-edges-rel-tacred   \n",
       "241  base  None                             glove-edges-rel-tacred   \n",
       "242  base  None                        openai-cat-edges-rel-tacred   \n",
       "243  base  None                        openai-lex-edges-rel-tacred   \n",
       "244  base  None                        openai-mix-edges-rel-tacred   \n",
       "245   bow  None             bert-base-uncased-lex-edges-rel-tacred   \n",
       "246   bow  None            bert-large-uncased-lex-edges-rel-tacred   \n",
       "247   bow  None                        elmo-chars-edges-rel-tacred   \n",
       "248   bow  None                             glove-edges-rel-tacred   \n",
       "249   bow  None                        openai-lex-edges-rel-tacred   \n",
       "250  base  None            bert-base-uncased-cat-edges-rel-semeval   \n",
       "251  base  None            bert-base-uncased-lex-edges-rel-semeval   \n",
       "252  base  None            bert-base-uncased-mix-edges-rel-semeval   \n",
       "253  base  None           bert-large-uncased-cat-edges-rel-semeval   \n",
       "254  base  None           bert-large-uncased-lex-edges-rel-semeval   \n",
       "255  base  None           bert-large-uncased-mix-edges-rel-semeval   \n",
       "256  base  None                             cove-edges-rel-semeval   \n",
       "257  base  None                       elmo-chars-edges-rel-semeval   \n",
       "258  base  None                        elmo-full-edges-rel-semeval   \n",
       "259  base  None                       elmo-ortho-edges-rel-semeval   \n",
       "260  base  None                            glove-edges-rel-semeval   \n",
       "261  base  None                       openai-cat-edges-rel-semeval   \n",
       "262  base  None                       openai-lex-edges-rel-semeval   \n",
       "263  base  None                       openai-mix-edges-rel-semeval   \n",
       "264   bow  None            bert-base-uncased-lex-edges-rel-semeval   \n",
       "265   bow  None           bert-large-uncased-lex-edges-rel-semeval   \n",
       "266   bow  None                       elmo-chars-edges-rel-semeval   \n",
       "267   bow  None                            glove-edges-rel-semeval   \n",
       "268   bow  None                       openai-lex-edges-rel-semeval   \n",
       "\n",
       "                   exp_type          label  num_epochs  num_steps  \\\n",
       "0                openai-cat              1         NaN        NaN   \n",
       "1                openai-lex              1         NaN        NaN   \n",
       "2                openai-bwb              1         NaN        NaN   \n",
       "3                      cove              1         NaN        NaN   \n",
       "4                elmo-chars              1         NaN        NaN   \n",
       "5                 elmo-full              1         NaN        NaN   \n",
       "6                elmo-ortho              1         NaN        NaN   \n",
       "7                     glove              1         NaN        NaN   \n",
       "8     bert-base-uncased-cat              1         NaN        NaN   \n",
       "9     bert-base-uncased-lex              1         NaN        NaN   \n",
       "10   bert-large-uncased-cat              1         NaN        NaN   \n",
       "11   bert-large-uncased-lex              1         NaN        NaN   \n",
       "12    bert-base-uncased-mix              1         NaN        NaN   \n",
       "13   bert-large-uncased-mix              1         NaN        NaN   \n",
       "14               openai-mix              1         NaN        NaN   \n",
       "15    bert-base-uncased-lex              1         NaN        NaN   \n",
       "16   bert-large-uncased-lex              1         NaN        NaN   \n",
       "17               elmo-chars              1         NaN        NaN   \n",
       "18                    glove              1         NaN        NaN   \n",
       "19               openai-lex              1         NaN        NaN   \n",
       "20               elmo-chars              1         NaN        NaN   \n",
       "21               elmo-chars              1         NaN        NaN   \n",
       "22               openai-cat    _micro_avg_         NaN        NaN   \n",
       "23               openai-lex    _micro_avg_         NaN        NaN   \n",
       "24               openai-bwb    _micro_avg_         NaN        NaN   \n",
       "25                     cove    _micro_avg_         NaN        NaN   \n",
       "26               elmo-chars    _micro_avg_         NaN        NaN   \n",
       "27                elmo-full    _micro_avg_         NaN        NaN   \n",
       "28               elmo-ortho    _micro_avg_         NaN        NaN   \n",
       "29                    glove    _micro_avg_         NaN        NaN   \n",
       "..                      ...            ...         ...        ...   \n",
       "239               elmo-full  _clean_micro_         NaN        NaN   \n",
       "240              elmo-ortho  _clean_micro_         NaN        NaN   \n",
       "241                   glove  _clean_micro_         NaN        NaN   \n",
       "242              openai-cat  _clean_micro_         NaN        NaN   \n",
       "243              openai-lex  _clean_micro_         NaN        NaN   \n",
       "244              openai-mix  _clean_micro_         NaN        NaN   \n",
       "245   bert-base-uncased-lex  _clean_micro_         NaN        NaN   \n",
       "246  bert-large-uncased-lex  _clean_micro_         NaN        NaN   \n",
       "247              elmo-chars  _clean_micro_         NaN        NaN   \n",
       "248                   glove  _clean_micro_         NaN        NaN   \n",
       "249              openai-lex  _clean_micro_         NaN        NaN   \n",
       "250   bert-base-uncased-cat  _clean_micro_         NaN        NaN   \n",
       "251   bert-base-uncased-lex  _clean_micro_         NaN        NaN   \n",
       "252   bert-base-uncased-mix  _clean_micro_         NaN        NaN   \n",
       "253  bert-large-uncased-cat  _clean_micro_         NaN        NaN   \n",
       "254  bert-large-uncased-lex  _clean_micro_         NaN        NaN   \n",
       "255  bert-large-uncased-mix  _clean_micro_         NaN        NaN   \n",
       "256                    cove  _clean_micro_         NaN        NaN   \n",
       "257              elmo-chars  _clean_micro_         NaN        NaN   \n",
       "258               elmo-full  _clean_micro_         NaN        NaN   \n",
       "259              elmo-ortho  _clean_micro_         NaN        NaN   \n",
       "260                   glove  _clean_micro_         NaN        NaN   \n",
       "261              openai-cat  _clean_micro_         NaN        NaN   \n",
       "262              openai-lex  _clean_micro_         NaN        NaN   \n",
       "263              openai-mix  _clean_micro_         NaN        NaN   \n",
       "264   bert-base-uncased-lex  _clean_micro_         NaN        NaN   \n",
       "265  bert-large-uncased-lex  _clean_micro_         NaN        NaN   \n",
       "266              elmo-chars  _clean_micro_         NaN        NaN   \n",
       "267                   glove  _clean_micro_         NaN        NaN   \n",
       "268              openai-lex  _clean_micro_         NaN        NaN   \n",
       "\n",
       "                                                   run                   task  \\\n",
       "0    /nfs/jsalt/home/iftenney/exp/edges-20180921-op...  coref-ontonotes-conll   \n",
       "1    /nfs/jsalt/home/iftenney/exp/edges-20180921-op...  coref-ontonotes-conll   \n",
       "2    /nfs/jsalt/home/iftenney/exp/edges-20180922-op...  coref-ontonotes-conll   \n",
       "3    /nfs/jsalt/home/iftenney/exp/final_20180927/ba...  coref-ontonotes-conll   \n",
       "4    /nfs/jsalt/home/iftenney/exp/final_20180927/ba...  coref-ontonotes-conll   \n",
       "5    /nfs/jsalt/home/iftenney/exp/final_20180927/ba...  coref-ontonotes-conll   \n",
       "6    /nfs/jsalt/home/iftenney/exp/final_20180927/ba...  coref-ontonotes-conll   \n",
       "7    /nfs/jsalt/home/iftenney/exp/final_20180927/ba...  coref-ontonotes-conll   \n",
       "8    /nfs/jsalt/exp/edges-20190124-bert/bert-base-u...  coref-ontonotes-conll   \n",
       "9    /nfs/jsalt/exp/edges-20190124-bert/bert-base-u...  coref-ontonotes-conll   \n",
       "10   /nfs/jsalt/exp/edges-20190125-bert-large/bert-...  coref-ontonotes-conll   \n",
       "11   /nfs/jsalt/exp/edges-20190125-bert-large/bert-...  coref-ontonotes-conll   \n",
       "12   /nfs/jsalt/exp/edges-20190129-bert-mix/bert-ba...  coref-ontonotes-conll   \n",
       "13   /nfs/jsalt/exp/edges-20190129-bert-mix/bert-la...  coref-ontonotes-conll   \n",
       "14   /nfs/jsalt/exp/edges-20190209-openai-mix/opena...  coref-ontonotes-conll   \n",
       "15   /nfs/jsalt/exp/edges-20190220-core-bow/bert-ba...  coref-ontonotes-conll   \n",
       "16   /nfs/jsalt/exp/edges-20190220-core-bow/bert-la...  coref-ontonotes-conll   \n",
       "17   /nfs/jsalt/exp/edges-20190220-core-bow/elmo-ch...  coref-ontonotes-conll   \n",
       "18   /nfs/jsalt/exp/edges-20190220-core-bow/glove-e...  coref-ontonotes-conll   \n",
       "19   /nfs/jsalt/exp/edges-20190220-core-bow/openai-...  coref-ontonotes-conll   \n",
       "20   /nfs/jsalt/home/iftenney/exp/final_20180927/cn...  coref-ontonotes-conll   \n",
       "21   /nfs/jsalt/home/iftenney/exp/final_20180927/cn...  coref-ontonotes-conll   \n",
       "22   /nfs/jsalt/home/iftenney/exp/edges-20180921-op...       dep-labeling-ewt   \n",
       "23   /nfs/jsalt/home/iftenney/exp/edges-20180921-op...       dep-labeling-ewt   \n",
       "24   /nfs/jsalt/home/iftenney/exp/edges-20180922-op...       dep-labeling-ewt   \n",
       "25   /nfs/jsalt/home/iftenney/exp/final_20180927/ba...       dep-labeling-ewt   \n",
       "26   /nfs/jsalt/home/iftenney/exp/final_20180927/ba...       dep-labeling-ewt   \n",
       "27   /nfs/jsalt/home/iftenney/exp/final_20180927/ba...       dep-labeling-ewt   \n",
       "28   /nfs/jsalt/home/iftenney/exp/final_20180927/ba...       dep-labeling-ewt   \n",
       "29   /nfs/jsalt/home/iftenney/exp/final_20180927/ba...       dep-labeling-ewt   \n",
       "..                                                 ...                    ...   \n",
       "239  /nfs/jsalt/exp/edges-20190205-tacred/elmo-full...             rel-tacred   \n",
       "240  /nfs/jsalt/exp/edges-20190205-tacred/elmo-orth...             rel-tacred   \n",
       "241  /nfs/jsalt/exp/edges-20190205-tacred/glove-edg...             rel-tacred   \n",
       "242  /nfs/jsalt/exp/edges-20190205-tacred/openai-ca...             rel-tacred   \n",
       "243  /nfs/jsalt/exp/edges-20190205-tacred/openai-le...             rel-tacred   \n",
       "244  /nfs/jsalt/exp/edges-20190209-openai-mix/opena...             rel-tacred   \n",
       "245  /nfs/jsalt/exp/edges-20190220-rel-bow//bert-ba...             rel-tacred   \n",
       "246  /nfs/jsalt/exp/edges-20190220-rel-bow//bert-la...             rel-tacred   \n",
       "247  /nfs/jsalt/exp/edges-20190220-rel-bow//elmo-ch...             rel-tacred   \n",
       "248  /nfs/jsalt/exp/edges-20190220-rel-bow//glove-e...             rel-tacred   \n",
       "249  /nfs/jsalt/exp/edges-20190220-rel-bow//openai-...             rel-tacred   \n",
       "250  /nfs/jsalt/exp/edges-20190205-semeval/bert-bas...            rel-semeval   \n",
       "251  /nfs/jsalt/exp/edges-20190205-semeval/bert-bas...            rel-semeval   \n",
       "252  /nfs/jsalt/exp/edges-20190205-semeval/bert-bas...            rel-semeval   \n",
       "253  /nfs/jsalt/exp/edges-20190205-semeval/bert-lar...            rel-semeval   \n",
       "254  /nfs/jsalt/exp/edges-20190205-semeval/bert-lar...            rel-semeval   \n",
       "255  /nfs/jsalt/exp/edges-20190205-semeval/bert-lar...            rel-semeval   \n",
       "256  /nfs/jsalt/exp/edges-20190205-semeval/cove-edg...            rel-semeval   \n",
       "257  /nfs/jsalt/exp/edges-20190205-semeval/elmo-cha...            rel-semeval   \n",
       "258  /nfs/jsalt/exp/edges-20190205-semeval/elmo-ful...            rel-semeval   \n",
       "259  /nfs/jsalt/exp/edges-20190205-semeval/elmo-ort...            rel-semeval   \n",
       "260  /nfs/jsalt/exp/edges-20190205-semeval/glove-ed...            rel-semeval   \n",
       "261  /nfs/jsalt/exp/edges-20190205-semeval/openai-c...            rel-semeval   \n",
       "262  /nfs/jsalt/exp/edges-20190205-semeval/openai-l...            rel-semeval   \n",
       "263  /nfs/jsalt/exp/edges-20190209-openai-mix/opena...            rel-semeval   \n",
       "264  /nfs/jsalt/exp/edges-20190220-rel-bow//bert-ba...            rel-semeval   \n",
       "265  /nfs/jsalt/exp/edges-20190220-rel-bow//bert-la...            rel-semeval   \n",
       "266  /nfs/jsalt/exp/edges-20190220-rel-bow//elmo-ch...            rel-semeval   \n",
       "267  /nfs/jsalt/exp/edges-20190220-rel-bow//glove-e...            rel-semeval   \n",
       "268  /nfs/jsalt/exp/edges-20190220-rel-bow//openai-...            rel-semeval   \n",
       "\n",
       "    split              ...               precision    recall  f1_score  \\\n",
       "0     val              ...                0.855263  0.817258  0.835829   \n",
       "1     val              ...                0.808771  0.660696  0.727273   \n",
       "2     val              ...                0.855844  0.810885  0.832758   \n",
       "3     val              ...                0.835032  0.761970  0.796830   \n",
       "4     val              ...                0.802593  0.714261  0.755855   \n",
       "5     val              ...                0.861156  0.834309  0.847520   \n",
       "6     val              ...                0.812175  0.781261  0.796418   \n",
       "7     val              ...                0.793172  0.684292  0.734720   \n",
       "8     val              ...                0.892236  0.892697  0.892467   \n",
       "9     val              ...                0.797980  0.721151  0.757622   \n",
       "10    val              ...                0.894575  0.903204  0.898869   \n",
       "11    val              ...                0.803609  0.720978  0.760054   \n",
       "12    val              ...                0.905996  0.903031  0.904511   \n",
       "13    val              ...                0.920096  0.924216  0.922152   \n",
       "14    val              ...                0.861145  0.863073  0.862108   \n",
       "15    val              ...                0.802081  0.756631  0.778694   \n",
       "16    val              ...                0.804084  0.752842  0.777620   \n",
       "17    val              ...                0.808431  0.746469  0.776216   \n",
       "18    val              ...                0.808152  0.734240  0.769425   \n",
       "19    val              ...                0.820021  0.670169  0.737560   \n",
       "20    val              ...                0.824132  0.776438  0.799574   \n",
       "21    val              ...                0.826167  0.774371  0.799431   \n",
       "22    val              ...                0.937647  0.904301  0.920672   \n",
       "23    val              ...                0.887963  0.691557  0.777549   \n",
       "24    val              ...                0.938856  0.905018  0.921626   \n",
       "25    val              ...                0.924871  0.878057  0.900856   \n",
       "26    val              ...                0.898903  0.730864  0.806221   \n",
       "27    val              ...                0.957174  0.927479  0.942093   \n",
       "28    val              ...                0.912583  0.800319  0.852772   \n",
       "29    val              ...                0.899781  0.671485  0.769048   \n",
       "..    ...              ...                     ...       ...       ...   \n",
       "239   val              ...                0.705402  0.413171  0.521114   \n",
       "240   val              ...                0.614417  0.233628  0.338531   \n",
       "241   val              ...                0.612549  0.174209  0.271269   \n",
       "242   val              ...                0.707666  0.309051  0.430218   \n",
       "243   val              ...                0.579454  0.132818  0.216103   \n",
       "244   val              ...                0.682254  0.327447  0.442511   \n",
       "245   val              ...                0.646013  0.247425  0.357808   \n",
       "246   val              ...                0.650498  0.180096  0.282092   \n",
       "247   val              ...                0.639163  0.286424  0.395579   \n",
       "248   val              ...                0.635789  0.258830  0.367891   \n",
       "249   val              ...                0.617951  0.241906  0.347700   \n",
       "250   val              ...                0.891473  0.718002  0.795389   \n",
       "251   val              ...                0.754653  0.464100  0.574742   \n",
       "252   val              ...                0.891667  0.779396  0.831760   \n",
       "253   val              ...                0.876448  0.708637  0.783659   \n",
       "254   val              ...                0.757774  0.481790  0.589059   \n",
       "255   val              ...                0.894366  0.792924  0.840596   \n",
       "256   val              ...                0.871693  0.685744  0.767618   \n",
       "257   val              ...                0.704274  0.428720  0.532988   \n",
       "258   val              ...                0.869001  0.697190  0.773672   \n",
       "259   val              ...                0.744863  0.452653  0.563107   \n",
       "260   val              ...                0.780847  0.441207  0.563830   \n",
       "261   val              ...                0.871043  0.773153  0.819184   \n",
       "262   val              ...                0.765000  0.477627  0.588085   \n",
       "263   val              ...                0.855056  0.791883  0.822258   \n",
       "264   val              ...                0.813738  0.640999  0.717113   \n",
       "265   val              ...                0.786260  0.643080  0.707499   \n",
       "266   val              ...                0.802372  0.633715  0.708140   \n",
       "267   val              ...                0.806242  0.645161  0.716763   \n",
       "268   val              ...                0.825166  0.648283  0.726107   \n",
       "\n",
       "     accuracy_errn95 precision_errn95 recall_errn95 f1_errn95     score  \\\n",
       "0           0.003098         0.009258      0.009941  0.009587  0.835829   \n",
       "1           0.003768         0.011192      0.012179  0.011665  0.727273   \n",
       "2           0.003118         0.009282      0.010073  0.009661  0.832758   \n",
       "3           0.003380         0.009994      0.010955  0.010452  0.796830   \n",
       "4           0.003651         0.010853      0.011621  0.011224  0.755855   \n",
       "5           0.003003         0.009036      0.009564  0.009293  0.847520   \n",
       "6           0.003423         0.010243      0.010634  0.010435  0.796418   \n",
       "7           0.003763         0.011217      0.011956  0.011575  0.734720   \n",
       "8           0.002567         0.007974      0.007961  0.007968  0.892467   \n",
       "9           0.003651         0.010864      0.011535  0.011189  0.757622   \n",
       "10          0.002499         0.007862      0.007606  0.007732  0.898869   \n",
       "11          0.003629         0.010789      0.011537  0.011150  0.760054   \n",
       "12          0.002424         0.007519      0.007612  0.007565  0.904511   \n",
       "13          0.002201         0.006959      0.006808  0.006882  0.922152   \n",
       "14          0.002888         0.008885      0.008843  0.008864  0.862108   \n",
       "15          0.003539         0.010552      0.011038  0.010790  0.778694   \n",
       "16          0.003541         0.010551      0.011096  0.010817  0.777620   \n",
       "17          0.003540         0.010535      0.011190  0.010853  0.776216   \n",
       "18          0.003575         0.010626      0.011363  0.010982  0.769425   \n",
       "19          0.003705         0.010931      0.012094  0.011483  0.737560   \n",
       "20          0.003383         0.010089      0.010717  0.010394  0.799574   \n",
       "21          0.003380         0.010069      0.010752  0.010399  0.799431   \n",
       "22          0.000098         0.003045      0.003639  0.003316  0.920672   \n",
       "23          0.000155         0.004421      0.005713  0.004984  0.777549   \n",
       "24          0.000097         0.003018      0.003626  0.003295  0.921626   \n",
       "25          0.000109         0.003346      0.004047  0.003664  0.900856   \n",
       "26          0.000146         0.004135      0.005486  0.004716  0.806221   \n",
       "27          0.000083         0.002544      0.003208  0.002838  0.942093   \n",
       "28          0.000130         0.003731      0.004945  0.004253  0.852772   \n",
       "29          0.000156         0.004300      0.005809  0.004942  0.769048   \n",
       "..               ...              ...           ...       ...       ...   \n",
       "239         0.000135         0.015834      0.013090  0.014332  0.521114   \n",
       "240         0.000148         0.020983      0.011249  0.014646  0.338531   \n",
       "241         0.000150         0.024285      0.010083  0.014249  0.271269   \n",
       "242         0.000141         0.018297      0.012284  0.014699  0.430218   \n",
       "243         0.000152         0.027410      0.009022  0.013576  0.216103   \n",
       "244         0.000141         0.017866      0.012475  0.014692  0.442511   \n",
       "245         0.000146         0.020541      0.011471  0.014721  0.357808   \n",
       "246         0.000149         0.024090      0.010215  0.014347  0.282092   \n",
       "247         0.000145         0.019071      0.012018  0.014745  0.395579   \n",
       "248         0.000146         0.020049      0.011643  0.014732  0.367891   \n",
       "249         0.000148         0.020645      0.011384  0.014676  0.347700   \n",
       "250         0.001770         0.021913      0.028450  0.024757  0.795389   \n",
       "251         0.002395         0.034692      0.031531  0.033036  0.574742   \n",
       "252         0.001637         0.021018      0.026217  0.023332  0.831760   \n",
       "253         0.001821         0.023138      0.028729  0.025632  0.783659   \n",
       "254         0.002371         0.033972      0.031592  0.032739  0.589059   \n",
       "255         0.001600         0.020639      0.025620  0.022861  0.840596   \n",
       "256         0.001875         0.023840      0.029351  0.026310  0.767618   \n",
       "257         0.002502         0.036982      0.031290  0.033899  0.532988   \n",
       "258         0.001858         0.023816      0.029051  0.026174  0.773672   \n",
       "259         0.002422         0.035357      0.031471  0.033301  0.563107   \n",
       "260         0.002388         0.034795      0.031394  0.033007  0.563830   \n",
       "261         0.001703         0.022492      0.026479  0.024323  0.819184   \n",
       "262         0.002365         0.033927      0.031581  0.032712  0.588085   \n",
       "263         0.001705         0.023129      0.025667  0.024332  0.822258   \n",
       "264         0.002065         0.027734      0.030330  0.028974  0.717113   \n",
       "265         0.002116         0.028660      0.030291  0.029453  0.707499   \n",
       "266         0.002097         0.028330      0.030461  0.029357  0.708140   \n",
       "267         0.002073         0.027935      0.030251  0.029047  0.716763   \n",
       "268         0.002031         0.027094      0.030191  0.028558  0.726107   \n",
       "\n",
       "     score_errn95                   display_row  \n",
       "0        0.009587       coref-ontonotes-conll-1  \n",
       "1        0.011665       coref-ontonotes-conll-1  \n",
       "2        0.009661       coref-ontonotes-conll-1  \n",
       "3        0.010452       coref-ontonotes-conll-1  \n",
       "4        0.011224       coref-ontonotes-conll-1  \n",
       "5        0.009293       coref-ontonotes-conll-1  \n",
       "6        0.010435       coref-ontonotes-conll-1  \n",
       "7        0.011575       coref-ontonotes-conll-1  \n",
       "8        0.007968       coref-ontonotes-conll-1  \n",
       "9        0.011189       coref-ontonotes-conll-1  \n",
       "10       0.007732       coref-ontonotes-conll-1  \n",
       "11       0.011150       coref-ontonotes-conll-1  \n",
       "12       0.007565       coref-ontonotes-conll-1  \n",
       "13       0.006882       coref-ontonotes-conll-1  \n",
       "14       0.008864       coref-ontonotes-conll-1  \n",
       "15       0.010790       coref-ontonotes-conll-1  \n",
       "16       0.010817       coref-ontonotes-conll-1  \n",
       "17       0.010853       coref-ontonotes-conll-1  \n",
       "18       0.010982       coref-ontonotes-conll-1  \n",
       "19       0.011483       coref-ontonotes-conll-1  \n",
       "20       0.010394       coref-ontonotes-conll-1  \n",
       "21       0.010399       coref-ontonotes-conll-1  \n",
       "22       0.003316  dep-labeling-ewt-_micro_avg_  \n",
       "23       0.004984  dep-labeling-ewt-_micro_avg_  \n",
       "24       0.003295  dep-labeling-ewt-_micro_avg_  \n",
       "25       0.003664  dep-labeling-ewt-_micro_avg_  \n",
       "26       0.004716  dep-labeling-ewt-_micro_avg_  \n",
       "27       0.002838  dep-labeling-ewt-_micro_avg_  \n",
       "28       0.004253  dep-labeling-ewt-_micro_avg_  \n",
       "29       0.004942  dep-labeling-ewt-_micro_avg_  \n",
       "..            ...                           ...  \n",
       "239      0.014332      rel-tacred-_clean_micro_  \n",
       "240      0.014646      rel-tacred-_clean_micro_  \n",
       "241      0.014249      rel-tacred-_clean_micro_  \n",
       "242      0.014699      rel-tacred-_clean_micro_  \n",
       "243      0.013576      rel-tacred-_clean_micro_  \n",
       "244      0.014692      rel-tacred-_clean_micro_  \n",
       "245      0.014721      rel-tacred-_clean_micro_  \n",
       "246      0.014347      rel-tacred-_clean_micro_  \n",
       "247      0.014745      rel-tacred-_clean_micro_  \n",
       "248      0.014732      rel-tacred-_clean_micro_  \n",
       "249      0.014676      rel-tacred-_clean_micro_  \n",
       "250      0.024757     rel-semeval-_clean_micro_  \n",
       "251      0.033036     rel-semeval-_clean_micro_  \n",
       "252      0.023332     rel-semeval-_clean_micro_  \n",
       "253      0.025632     rel-semeval-_clean_micro_  \n",
       "254      0.032739     rel-semeval-_clean_micro_  \n",
       "255      0.022861     rel-semeval-_clean_micro_  \n",
       "256      0.026310     rel-semeval-_clean_micro_  \n",
       "257      0.033899     rel-semeval-_clean_micro_  \n",
       "258      0.026174     rel-semeval-_clean_micro_  \n",
       "259      0.033301     rel-semeval-_clean_micro_  \n",
       "260      0.033007     rel-semeval-_clean_micro_  \n",
       "261      0.024323     rel-semeval-_clean_micro_  \n",
       "262      0.032712     rel-semeval-_clean_micro_  \n",
       "263      0.024332     rel-semeval-_clean_micro_  \n",
       "264      0.028974     rel-semeval-_clean_micro_  \n",
       "265      0.029453     rel-semeval-_clean_micro_  \n",
       "266      0.029357     rel-semeval-_clean_micro_  \n",
       "267      0.029047     rel-semeval-_clean_micro_  \n",
       "268      0.028558     rel-semeval-_clean_micro_  \n",
       "\n",
       "[269 rows x 31 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SPLIT = \"test\"\n",
    "SPLIT = \"val\"\n",
    "mask = df['split'] == SPLIT\n",
    "mask &= (df['exp_type'] != \"openai\")\n",
    "mask &= df['exp_type'].map(lambda s: '-cased-' not in s)  # skip cased BERT for now\n",
    "mask &= (df['task'] != 'dpr') | df['seed'].notnull()  # only use folds for DPR\n",
    "# Skip these tasks\n",
    "mask &= (df['task'] != \"constituent-ontonotes\")\n",
    "mask &= (df['task'] != \"ner-tacred\")\n",
    "mask &= (df['task'] != \"coref-gap\")\n",
    "mask &= (df['task'] != \"coref-gap-ontonotes\")\n",
    "mask &= (df['task'] != \"noun-verb\")\n",
    "# mask &= (df['task'] != \"rel-tacred\")\n",
    "# mask &= (df['task'] != \"rel-semeval\")\n",
    "\n",
    "final_scores = []\n",
    "for task in df['task'].unique():\n",
    "    task_scores = df[mask & (df['task'] == task)]\n",
    "    if is_coref_task(task):\n",
    "        final_scores.append(task_scores[task_scores['label'] == \"1\"])\n",
    "        # For GAP coref, have stratified by gender\n",
    "        if task.startswith(\"coref-gap\"):\n",
    "            final_scores.append(task_scores[task_scores['label'] == \"_info.pronoun_gender_MASCULINE_1_\"])\n",
    "            final_scores.append(task_scores[task_scores['label'] == \"_info.pronoun_gender_FEMININE_1_\"])\n",
    "    elif task == \"dpr\":\n",
    "        dpr_mask = task_scores['seed'] == \"_mean_\"\n",
    "        dpr_mask &= task_scores['label'] == \"_micro_avg_\"\n",
    "        final_scores.append(task_scores[dpr_mask])\n",
    "    elif is_srl_task(task):\n",
    "        final_scores.append(task_scores[task_scores['label'] == '_core_'])\n",
    "        final_scores.append(task_scores[task_scores['label'] == '_non_core_'])\n",
    "        # Use clean version, average only over core or noncore roles.\n",
    "        final_scores.append(task_scores[task_scores['label'] == '_clean_micro_'])\n",
    "    elif is_constituent_task(task):\n",
    "        final_scores.append(task_scores[task_scores['label'] == '_pos_'])\n",
    "        final_scores.append(task_scores[task_scores['label'] == '_nonterminal_'])\n",
    "        final_scores.append(task_scores[task_scores['label'] == '_micro_avg_'])\n",
    "    elif is_relation_task(task):\n",
    "        # Relation tasks include specific \"no_relation\" label\n",
    "        final_scores.append(task_scores[task_scores['label'] == '_clean_micro_'])\n",
    "    elif task == \"noun-verb\":\n",
    "        # Noun-verb reports accuracy on VERB class\n",
    "        final_scores.append(task_scores[task_scores['label'] == 'VERB'])\n",
    "    else:\n",
    "        final_scores.append(task_scores[task_scores['label'] == '_micro_avg_'])\n",
    "        \n",
    "fdf = pd.concat(final_scores, axis=0, ignore_index=True, sort=False)\n",
    "# fdf['task_and_metric'] = [\"%s-%s\" % tl for tl in zip(fdf.task, fdf.label)]\n",
    "def format_display_row(task, label, seed):\n",
    "    ret = f\"{task}-{label}\"\n",
    "    if seed:\n",
    "        ret += f\":{seed}\"\n",
    "    return ret\n",
    "\n",
    "fdf['display_row'] = [format_display_row(*args) for args in zip(fdf.task, fdf.label, fdf.seed)]\n",
    "print(len(fdf))\n",
    "fdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make Table 2 (model comparison)\n",
    "\n",
    "This should get us 90% of the way there with formatting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">BERT (base)</th>\n",
       "      <th colspan=\"5\" halign=\"left\">BERT (large)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">F1 Score</th>\n",
       "      <th>Abs. $\\Delta$</th>\n",
       "      <th colspan=\"3\" halign=\"left\">F1 Score</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Abs. $\\Delta$</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Lex.</th>\n",
       "      <th>\\texttt{cat}</th>\n",
       "      <th>\\texttt{mix}</th>\n",
       "      <th>ELMo</th>\n",
       "      <th>Lex.</th>\n",
       "      <th>\\texttt{cat}</th>\n",
       "      <th>\\texttt{mix}</th>\n",
       "      <th>(base)</th>\n",
       "      <th>ELMo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Part-of-Speech</th>\n",
       "      <td>88.3</td>\n",
       "      <td>\\textbf{96.8}</td>\n",
       "      <td>96.5</td>\n",
       "      <td>-0.1</td>\n",
       "      <td>88.0</td>\n",
       "      <td>96.3</td>\n",
       "      <td>\\textbf{96.6}</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Constituents</th>\n",
       "      <td>66.9</td>\n",
       "      <td>83.5</td>\n",
       "      <td>86.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>67.8</td>\n",
       "      <td>79.6</td>\n",
       "      <td>\\textbf{86.9}</td>\n",
       "      <td>0.4</td>\n",
       "      <td>2.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dependencies</th>\n",
       "      <td>80.2</td>\n",
       "      <td>93.4</td>\n",
       "      <td>95.4</td>\n",
       "      <td>1.2</td>\n",
       "      <td>80.3</td>\n",
       "      <td>92.1</td>\n",
       "      <td>\\textbf{95.6}</td>\n",
       "      <td>0.3</td>\n",
       "      <td>1.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Entities</th>\n",
       "      <td>89.7</td>\n",
       "      <td>95.9</td>\n",
       "      <td>96.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>90.7</td>\n",
       "      <td>96.0</td>\n",
       "      <td>\\textbf{96.5}</td>\n",
       "      <td>0.4</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SRL (all)</th>\n",
       "      <td>74.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>90.9</td>\n",
       "      <td>1.1</td>\n",
       "      <td>75.1</td>\n",
       "      <td>87.7</td>\n",
       "      <td>\\textbf{92.0}</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\quad Core roles</th>\n",
       "      <td>\\textit{73.3}</td>\n",
       "      <td>\\textit{90.7}</td>\n",
       "      <td>\\textit{93.0}</td>\n",
       "      <td>\\textit{0.8}</td>\n",
       "      <td>\\textit{74.5}</td>\n",
       "      <td>\\textit{89.2}</td>\n",
       "      <td>\\textbf{\\textit{94.1}}</td>\n",
       "      <td>\\textit{1.1}</td>\n",
       "      <td>\\textit{1.9}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\quad Non-core roles</th>\n",
       "      <td>\\textit{76.0}</td>\n",
       "      <td>\\textit{84.6}</td>\n",
       "      <td>\\textit{85.8}</td>\n",
       "      <td>\\textit{1.6}</td>\n",
       "      <td>\\textit{76.4}</td>\n",
       "      <td>\\textit{83.9}</td>\n",
       "      <td>\\textbf{\\textit{86.8}}</td>\n",
       "      <td>\\textit{1.0}</td>\n",
       "      <td>\\textit{2.6}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OntoNotes coref.</th>\n",
       "      <td>75.8</td>\n",
       "      <td>89.2</td>\n",
       "      <td>90.5</td>\n",
       "      <td>5.7</td>\n",
       "      <td>76.0</td>\n",
       "      <td>89.9</td>\n",
       "      <td>\\textbf{92.2}</td>\n",
       "      <td>1.8</td>\n",
       "      <td>7.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SPR1</th>\n",
       "      <td>76.7</td>\n",
       "      <td>83.3</td>\n",
       "      <td>\\textbf{84.0}</td>\n",
       "      <td>1.1</td>\n",
       "      <td>76.7</td>\n",
       "      <td>82.3</td>\n",
       "      <td>\\textbf{83.7}</td>\n",
       "      <td>-0.3</td>\n",
       "      <td>0.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SPR2</th>\n",
       "      <td>80.4</td>\n",
       "      <td>\\textbf{82.6}</td>\n",
       "      <td>\\textbf{82.8}</td>\n",
       "      <td>0.3</td>\n",
       "      <td>80.5</td>\n",
       "      <td>82.3</td>\n",
       "      <td>\\textbf{82.9}</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Winograd coref.</th>\n",
       "      <td>53.9</td>\n",
       "      <td>54.9</td>\n",
       "      <td>55.9</td>\n",
       "      <td>1.3</td>\n",
       "      <td>53.6</td>\n",
       "      <td>54.5</td>\n",
       "      <td>\\textbf{61.8}</td>\n",
       "      <td>5.8</td>\n",
       "      <td>7.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rel. (TACRED)</th>\n",
       "      <td>20.0</td>\n",
       "      <td>48.4</td>\n",
       "      <td>\\textbf{52.2}</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.7</td>\n",
       "      <td>48.1</td>\n",
       "      <td>\\textbf{51.9}</td>\n",
       "      <td>-0.2</td>\n",
       "      <td>-0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rel. (SemEval)</th>\n",
       "      <td>57.5</td>\n",
       "      <td>79.5</td>\n",
       "      <td>83.2</td>\n",
       "      <td>5.8</td>\n",
       "      <td>58.9</td>\n",
       "      <td>78.4</td>\n",
       "      <td>\\textbf{84.1}</td>\n",
       "      <td>0.9</td>\n",
       "      <td>6.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\\midrule Macro Average</th>\n",
       "      <td>69.4</td>\n",
       "      <td>81.5</td>\n",
       "      <td>83.1</td>\n",
       "      <td>1.7</td>\n",
       "      <td>70.4</td>\n",
       "      <td>80.6</td>\n",
       "      <td>\\textbf{84.0}</td>\n",
       "      <td>0.9</td>\n",
       "      <td>2.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          BERT (base)                                \\\n",
       "                             F1 Score                                 \n",
       "                                 Lex.   \\texttt{cat}   \\texttt{mix}   \n",
       "Part-of-Speech                   88.3  \\textbf{96.8}           96.5   \n",
       "Constituents                     66.9           83.5           86.6   \n",
       "Dependencies                     80.2           93.4           95.4   \n",
       "Entities                         89.7           95.9           96.0   \n",
       "SRL (all)                        74.0           89.0           90.9   \n",
       "\\quad Core roles        \\textit{73.3}  \\textit{90.7}  \\textit{93.0}   \n",
       "\\quad Non-core roles    \\textit{76.0}  \\textit{84.6}  \\textit{85.8}   \n",
       "OntoNotes coref.                 75.8           89.2           90.5   \n",
       "SPR1                             76.7           83.3  \\textbf{84.0}   \n",
       "SPR2                             80.4  \\textbf{82.6}  \\textbf{82.8}   \n",
       "Winograd coref.                  53.9           54.9           55.9   \n",
       "Rel. (TACRED)                    20.0           48.4  \\textbf{52.2}   \n",
       "Rel. (SemEval)                   57.5           79.5           83.2   \n",
       "\\midrule Macro Average           69.4           81.5           83.1   \n",
       "\n",
       "                                       BERT (large)                 \\\n",
       "                       Abs. $\\Delta$       F1 Score                  \n",
       "                                ELMo           Lex.   \\texttt{cat}   \n",
       "Part-of-Speech                  -0.1           88.0           96.3   \n",
       "Constituents                     1.9           67.8           79.6   \n",
       "Dependencies                     1.2           80.3           92.1   \n",
       "Entities                         0.8           90.7           96.0   \n",
       "SRL (all)                        1.1           75.1           87.7   \n",
       "\\quad Core roles        \\textit{0.8}  \\textit{74.5}  \\textit{89.2}   \n",
       "\\quad Non-core roles    \\textit{1.6}  \\textit{76.4}  \\textit{83.9}   \n",
       "OntoNotes coref.                 5.7           76.0           89.9   \n",
       "SPR1                             1.1           76.7           82.3   \n",
       "SPR2                             0.3           80.5           82.3   \n",
       "Winograd coref.                  1.3           53.6           54.5   \n",
       "Rel. (TACRED)                    0.0           26.7           48.1   \n",
       "Rel. (SemEval)                   5.8           58.9           78.4   \n",
       "\\midrule Macro Average           1.7           70.4           80.6   \n",
       "\n",
       "                                                                            \n",
       "                                               Abs. $\\Delta$                \n",
       "                                  \\texttt{mix}        (base)          ELMo  \n",
       "Part-of-Speech                   \\textbf{96.6}           0.2           0.0  \n",
       "Constituents                     \\textbf{86.9}           0.4           2.2  \n",
       "Dependencies                     \\textbf{95.6}           0.3           1.4  \n",
       "Entities                         \\textbf{96.5}           0.4           1.2  \n",
       "SRL (all)                        \\textbf{92.0}           1.0           2.1  \n",
       "\\quad Core roles        \\textbf{\\textit{94.1}}  \\textit{1.1}  \\textit{1.9}  \n",
       "\\quad Non-core roles    \\textbf{\\textit{86.8}}  \\textit{1.0}  \\textit{2.6}  \n",
       "OntoNotes coref.                 \\textbf{92.2}           1.8           7.5  \n",
       "SPR1                             \\textbf{83.7}          -0.3           0.7  \n",
       "SPR2                             \\textbf{82.9}           0.1           0.4  \n",
       "Winograd coref.                  \\textbf{61.8}           5.8           7.1  \n",
       "Rel. (TACRED)                    \\textbf{51.9}          -0.2          -0.2  \n",
       "Rel. (SemEval)                   \\textbf{84.1}           0.9           6.7  \n",
       "\\midrule Macro Average           \\textbf{84.0}           0.9           2.7  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{@{}l|cccr|cccrr@{}}\n",
      "\\toprule\n",
      "{} & \\multicolumn{4}{c}{\\textbf{BERT (base)}} & \\multicolumn{5}{c}{\\textbf{BERT (large)}} \\\\\n",
      "{} & \\multicolumn{3}{c}{\\textbf{F1 Score}} & \\textbf{Abs. $\\Delta$} & \\multicolumn{3}{c}{\\textbf{F1 Score}} & \\multicolumn{2}{c}{\\textbf{Abs. $\\Delta$}} \\\\\n",
      "{} &        \\textbf{Lex.} & \\textbf{\\texttt{cat}} & \\textbf{\\texttt{mix}} &          \\textbf{ELMo} &         \\textbf{Lex.} & \\textbf{\\texttt{cat}} &   \\textbf{\\texttt{mix}} &        \\textbf{(base)} & \\textbf{ELMo} \\\\\n",
      "\\midrule\n",
      "Part-of-Speech         &                 88.3 &         \\textbf{96.8} &                  96.5 &                   -0.1 &                  88.0 &                  96.3 &           \\textbf{96.6} &                    0.2 &           0.0 \\\\\n",
      "Constituents           &                 66.9 &                  83.5 &                  86.6 &                    1.9 &                  67.8 &                  79.6 &           \\textbf{86.9} &                    0.4 &           2.2 \\\\\n",
      "Dependencies           &                 80.2 &                  93.4 &                  95.4 &                    1.2 &                  80.3 &                  92.1 &           \\textbf{95.6} &                    0.3 &           1.4 \\\\\n",
      "Entities               &                 89.7 &                  95.9 &                  96.0 &                    0.8 &                  90.7 &                  96.0 &           \\textbf{96.5} &                    0.4 &           1.2 \\\\\n",
      "SRL (all)              &                 74.0 &                  89.0 &                  90.9 &                    1.1 &                  75.1 &                  87.7 &           \\textbf{92.0} &                    1.0 &           2.1 \\\\\n",
      "\\quad Core roles       &        \\textit{73.3} &         \\textit{90.7} &         \\textit{93.0} &           \\textit{0.8} &         \\textit{74.5} &         \\textit{89.2} &  \\textbf{\\textit{94.1}} &           \\textit{1.1} &  \\textit{1.9} \\\\\n",
      "\\quad Non-core roles   &        \\textit{76.0} &         \\textit{84.6} &         \\textit{85.8} &           \\textit{1.6} &         \\textit{76.4} &         \\textit{83.9} &  \\textbf{\\textit{86.8}} &           \\textit{1.0} &  \\textit{2.6} \\\\\n",
      "OntoNotes coref.       &                 75.8 &                  89.2 &                  90.5 &                    5.7 &                  76.0 &                  89.9 &           \\textbf{92.2} &                    1.8 &           7.5 \\\\\n",
      "SPR1                   &                 76.7 &                  83.3 &         \\textbf{84.0} &                    1.1 &                  76.7 &                  82.3 &           \\textbf{83.7} &                   -0.3 &           0.7 \\\\\n",
      "SPR2                   &                 80.4 &         \\textbf{82.6} &         \\textbf{82.8} &                    0.3 &                  80.5 &                  82.3 &           \\textbf{82.9} &                    0.1 &           0.4 \\\\\n",
      "Winograd coref.        &                 53.9 &                  54.9 &                  55.9 &                    1.3 &                  53.6 &                  54.5 &           \\textbf{61.8} &                    5.8 &           7.1 \\\\\n",
      "Rel. (TACRED)          &                 20.0 &                  48.4 &         \\textbf{52.2} &                    0.0 &                  26.7 &                  48.1 &           \\textbf{51.9} &                   -0.2 &          -0.2 \\\\\n",
      "Rel. (SemEval)         &                 57.5 &                  79.5 &                  83.2 &                    5.8 &                  58.9 &                  78.4 &           \\textbf{84.1} &                    0.9 &           6.7 \\\\\n",
      "\\midrule Macro Average &                 69.4 &                  81.5 &                  83.1 &                    1.7 &                  70.4 &                  80.6 &           \\textbf{84.0} &                    0.9 &           2.7 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# scores and confidence intervals\n",
    "sdf = fdf.pivot(index=\"display_row\", columns=\"display_col\", values=\"score\")\n",
    "cdf = fdf.pivot(index=\"display_row\", columns=\"display_col\", values=\"score_errn95\")\n",
    "\n",
    "# Table 2, top part (original table)\n",
    "columns = collections.OrderedDict()\n",
    "columns[(\"CoVe\", \"Lex.\")] = sdf['glove (base)']\n",
    "columns[(\"CoVe\", \"Full\")] = sdf['cove (base)']\n",
    "columns[(\"CoVe\", \"Abs. $\\Delta$\")] = sdf['cove (base)'] - sdf['glove (base)']\n",
    "columns[(\"ELMo\", \"Lex.\")] = sdf['elmo-chars (base)']\n",
    "columns[(\"ELMo\", \"Full\")] = sdf['elmo-full (base)']\n",
    "columns[(\"ELMo\", \"Abs. $\\Delta$\")] = sdf['elmo-full (base)'] - sdf['elmo-chars (base)']\n",
    "columns[(\"Transformer LM\", \"Lex.\")] = sdf['openai-lex (base)']\n",
    "columns[(\"Transformer LM\", \"\\texttt{cat}\")] = sdf['openai-cat (base)']\n",
    "# columns[(\"Transformer LM\", \"Abs. $\\Delta$\")] = sdf['openai-cat (base)'] - sdf['openai-lex (base)']\n",
    "columns[(\"Transformer LM\", \"\\texttt{mix}\")] = sdf['openai-mix (base)']\n",
    "# columns[(\"Transformer LM\", \"Abs. $\\Delta$\")] = sdf['openai-mix (base)'] - sdf['openai-lex (base)']\n",
    "COLUMN_FORMAT=\"|ccr|ccr|ccc\"\n",
    "# Table 2, bottom part (new results)\n",
    "columns = collections.OrderedDict()\n",
    "columns[(\"BERT (base)\", \"F1 Score\", \"Lex.\")] = sdf['bert-base-uncased-lex (base)']\n",
    "columns[(\"BERT (base)\", \"F1 Score\", \"\\texttt{cat}\")] = sdf['bert-base-uncased-cat (base)']\n",
    "columns[(\"BERT (base)\", \"F1 Score\", \"\\texttt{mix}\")] = sdf['bert-base-uncased-mix (base)']\n",
    "# columns[(\"BERT (base)\", \"Abs. $\\Delta$\", \"Lex.\")] = sdf['bert-base-uncased-mix (base)'] - sdf['bert-base-uncased-lex (base)']\n",
    "columns[(\"BERT (base)\", \"Abs. $\\Delta$\", \"ELMo\")] = sdf['bert-base-uncased-mix (base)'] - sdf['elmo-full (base)']\n",
    "columns[(\"BERT (large)\", \"F1 Score\", \"Lex.\")] = sdf['bert-large-uncased-lex (base)']\n",
    "columns[(\"BERT (large)\", \"F1 Score\", \"\\texttt{cat}\")] = sdf['bert-large-uncased-cat (base)']\n",
    "columns[(\"BERT (large)\", \"F1 Score\", \"\\texttt{mix}\")] = sdf['bert-large-uncased-mix (base)']\n",
    "# columns[(\"BERT (large)\", \"Abs. $\\Delta$\", \"Lex.\")] = sdf['bert-large-uncased-mix (base)'] - sdf['bert-large-uncased-lex (base)']\n",
    "columns[(\"BERT (large)\", \"Abs. $\\Delta$\", \"(base)\")] = sdf['bert-large-uncased-mix (base)'] - sdf['bert-base-uncased-mix (base)']\n",
    "columns[(\"BERT (large)\", \"Abs. $\\Delta$\", \"ELMo\")] = sdf['bert-large-uncased-mix (base)'] - sdf['elmo-full (base)']\n",
    "COLUMN_FORMAT=\"|cccr|cccrr\"\n",
    "\n",
    "# Make a DataFrame that looks like the LaTeX table.\n",
    "pdf = pd.DataFrame(columns)\n",
    "pdf = pdf.reindex(sorted(pdf.index, key=task_sort_key), axis=0)\n",
    "\n",
    "# Compute macro average as series, with entries for each column\n",
    "def _is_core_task(row):\n",
    "    return not re.match(r'.*-_(non_)?core_$', row)\n",
    "core_task_mask = list(pdf.index.map(_is_core_task))\n",
    "macro_average = pdf[core_task_mask].mean()\n",
    "macro_average.name = \"Macro Average\"\n",
    "\n",
    "pdf.index = [_make_display_name(*row.rsplit(\"-\", 1)) for row in pdf.index]\n",
    "pdf = pdf.append(macro_average)\n",
    "\n",
    "# Get row maxima\n",
    "maxima = {}\n",
    "score_cols = [c for c in pdf.columns if not \"Delta\" in c[1]]\n",
    "for row in pdf.index:\n",
    "    maxima[row] = pdf.loc[row, score_cols].max()\n",
    "\n",
    "# Format all numbers into strings\n",
    "# _format_score_col = lambda f: \"{:.0f}\".format(100*f)\n",
    "_format_score_col = lambda f: \"{:.1f}\".format(100*f)\n",
    "_format_delta_col = lambda f: \"{:.1f}\".format(100*f)\n",
    "# def _get_margin_percent(row):\n",
    "#     if row.startswith(\"Winograd\"):\n",
    "#         return 3\n",
    "#     elif row.startswith(\"SPR\"):\n",
    "#         return 1\n",
    "#     else:\n",
    "#         return 0.5\n",
    "def _get_margin_percent(row):\n",
    "    if row.startswith(\"Winograd\"):\n",
    "        return 1.5\n",
    "    elif row.startswith(\"SPR\"):\n",
    "        return 0.5\n",
    "    else:\n",
    "        return 0.25\n",
    "\n",
    "\n",
    "def _format_cell(i, row, col, val):\n",
    "    num_val = val\n",
    "    if \"Delta\" in col[1]:\n",
    "        val = _format_delta_col(val)\n",
    "    elif row == \"Macro Average\":\n",
    "        val = _format_delta_col(val)\n",
    "    else:\n",
    "        val = _format_score_col(val)\n",
    "    if i < len(core_task_mask) and not core_task_mask[i]:\n",
    "        val = \"\\textit{\" + val + \"}\"\n",
    "#     if row.startswith(\"Winograd\") and col == (\"BERT (large)\", \"F1 Score\", \"\\texttt{mix}\"):\n",
    "#         val = val + \" $\\pm$ 6\"\n",
    "    margin = 0.01 * _get_margin_percent(row)\n",
    "    if num_val > maxima[row] - margin:\n",
    "        val = \"\\textbf{\" + val + \"}\"\n",
    "    return val\n",
    "    \n",
    "for i, row in enumerate(pdf.index):\n",
    "    for col in pdf.columns:\n",
    "        pdf.loc[row, col] = _format_cell(i, row, col, pdf.loc[row, col])\n",
    "        \n",
    "# Change some row labels\n",
    "def _rename_row(row):\n",
    "    if row == \"SRL (core)\":\n",
    "        return \"\\quad Core roles\"\n",
    "    elif row == \"SRL (non-core)\":\n",
    "        return \"\\quad Non-core roles\"\n",
    "    elif row.endswith(\" (mean)\"):\n",
    "        return row[:-7]\n",
    "    elif row.startswith(\"Relations \"):\n",
    "        return \"Rel. \" + row.split(\" \", 1)[1]\n",
    "    elif row == \"Macro Average\":\n",
    "        return \"\\midrule \" + row\n",
    "    else:\n",
    "        return row\n",
    "pdf.index = pdf.index.map(_rename_row)\n",
    "    \n",
    "display(pdf)\n",
    "        \n",
    "# Make columns bold.\n",
    "_make_bold = lambda text: \"\\textbf{\" + text + \"}\"\n",
    "pdf.columns = pdf.columns.map(lambda c: tuple(map(_make_bold, c)))\n",
    "\n",
    "tex = pdf.to_latex(column_format=\"@{}l\"+COLUMN_FORMAT+\"@{}\", float_format=\"%.2f\",\n",
    "                   bold_rows=False, escape=False, multicolumn=True, multicolumn_format=\"c\")\n",
    "print(tex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pivot DataFrame to present each task on a row, and each experiment on a column.\n",
    "\n",
    "This form is suitable to copy-paste into a spreadsheet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "display_row,glove (base),glove (bow),cove (base),elmo-chars (base),elmo-chars (bow),elmo-chars (cnn1),elmo-chars (cnn2),elmo-ortho (base),elmo-full (base),openai-lex (base),openai-lex (bow),openai-cat (base),openai-mix (base),openai-bwb (base),bert-base-uncased-lex (base),bert-base-uncased-lex (bow),bert-base-uncased-cat (base),bert-base-uncased-mix (base),bert-large-uncased-lex (base),bert-large-uncased-lex (bow),bert-large-uncased-cat (base),bert-large-uncased-mix (base)\n",
      "pos-ontonotes-_micro_avg_,85.9228,88.4670,93.9147,90.6979,90.8564,95.7660,96.0406,91.2232,96.6235,88.0053,90.2986,94.6398,94.6746,,88.3494,90.6203,96.7583,96.4810,87.9578,90.7934,96.2727,96.6480\n",
      "nonterminal-ontonotes-_micro_avg_,55.1509,58.9663,81.4808,68.7957,72.6901,83.9149,84.7074,71.0830,84.7232,64.3216,66.9262,80.9122,84.1387,,66.9288,70.6040,83.4807,86.5765,67.7886,71.4839,79.6116,86.9384\n",
      "dep-labeling-ewt-_micro_avg_,76.9048,79.0899,90.0856,80.6221,80.5477,90.7202,91.7454,85.2772,94.2093,77.7549,79.0374,92.0672,94.0847,92.1626,80.1903,81.6522,93.4106,95.3599,80.2943,81.8894,92.0770,95.6186\n",
      "ner-ontonotes-_micro_avg_,89.2697,91.3087,92.0233,91.3526,93.3942,93.4360,93.9312,92.1218,95.2299,87.8648,90.6177,93.2675,92.7911,94.6704,89.7127,92.1164,95.8886,96.0229,90.6850,91.9104,95.9514,96.4726\n",
      "srl-conll2012-_clean_micro_,57.1503,59.2388,79.9468,72.9396,74.4974,84.4006,85.9012,77.1182,89.8543,66.2599,68.8229,85.7341,89.2792,86.7214,74.0099,75.6584,88.9823,90.9143,75.0541,76.6083,87.7154,91.9638\n",
      "srl-conll2012-_core_,53.5800,56.2619,80.3947,72.3186,73.9859,86.1781,87.9790,77.4224,92.1271,63.4472,66.6411,87.3879,91.3408,88.7786,73.2534,75.5327,90.7377,92.9704,74.5300,76.4915,89.2066,94.0546\n",
      "srl-conll2012-_non_core_,65.9205,66.4651,78.8007,74.5405,75.8130,79.8228,80.5576,76.3248,84.1404,73.2789,74.3131,81.5030,84.1322,81.4171,75.9532,75.9820,84.5714,85.7894,76.4225,76.9067,83.9472,86.7640\n",
      "coref-ontonotes-conll-1,73.4720,76.9425,79.6830,75.5855,77.6216,79.9574,79.9431,79.6418,84.7520,72.7273,73.7560,83.5829,86.2108,83.2758,75.7622,77.8694,89.2467,90.4511,76.0054,77.7620,89.8869,92.2152\n",
      "spr1-_micro_avg_,76.3171,77.3620,79.6920,78.6297,79.1338,79.9119,79.8563,79.4347,82.9550,76.8418,77.2299,81.5752,81.5816,82.9812,76.6975,77.0549,83.3432,84.0343,76.6629,77.1346,82.2582,83.6857\n",
      "spr2-_micro_avg_,80.7270,80.9592,81.7515,81.2221,81.3423,80.9878,81.0411,81.4711,82.5456,80.6933,80.5784,82.2470,82.4704,82.4584,80.4497,80.7010,82.6340,82.8195,80.4791,80.6469,82.2516,82.9316\n",
      "dpr-_micro_avg_:_mean_,53.2291,,54.3673,54.8296,,,,,54.6406,53.3139,,51.9203,53.3875,,53.9219,,54.9189,55.9095,53.5780,,54.5451,61.7591\n",
      "rel-tacred-_clean_micro_,27.1269,36.7891,36.2348,33.1749,39.5579,,,33.8531,52.1114,21.6103,34.7700,43.0218,44.2511,,19.9784,35.7808,48.4036,52.1520,26.6874,28.2092,48.0512,51.9079\n",
      "rel-semeval-_clean_micro_,56.3830,71.6763,76.7618,53.2988,70.8140,,,56.3107,77.3672,58.8085,72.6107,81.9184,82.2258,,57.4742,71.7113,79.5389,83.1760,58.9059,70.7499,78.3659,84.0596\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Pivot to wide-form for spreadsheet, and sort in (mostly) stable order.\n",
    "# mask = fdf['task'].map(lambda s: s.startswith('rel-'))\n",
    "sheet_df = fdf.pivot(index=\"display_row\", columns=\"display_col\", values=\"score\")\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.columns, \n",
    "                                   key=exp_type_sort_key), axis=1)\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.index,\n",
    "                                   key=task_sort_key), axis=0)\n",
    "# sheet_df\n",
    "print((100*sheet_df).to_csv(**csv_args))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print the same format, but show the 95% confidence intervals for each score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "display_row,glove (base),glove (bow),cove (base),elmo-chars (base),elmo-chars (bow),elmo-chars (cnn1),elmo-chars (cnn2),elmo-ortho (base),elmo-full (base),openai-lex (base),openai-lex (bow),openai-cat (base),openai-mix (base),openai-bwb (base),bert-base-uncased-lex (base),bert-base-uncased-lex (bow),bert-base-uncased-cat (base),bert-base-uncased-mix (base),bert-large-uncased-lex (base),bert-large-uncased-lex (bow),bert-large-uncased-cat (base),bert-large-uncased-mix (base)\n",
      "pos-ontonotes-_micro_avg_,0.1253,0.1164,0.0850,0.1059,0.1051,0.0732,0.0707,0.1030,0.0656,0.1186,0.1078,0.0819,0.0818,,0.1171,0.1063,0.0644,0.0671,0.1188,0.1054,0.0690,0.0655\n",
      "nonterminal-ontonotes-_micro_avg_,0.2055,0.2014,0.1514,0.1847,0.1765,0.1434,0.1403,0.1792,0.1403,0.1925,0.1883,0.1538,0.1424,,0.1881,0.1808,0.1450,0.1325,0.1863,0.1787,0.1577,0.1310\n",
      "dep-labeling-ewt-_micro_avg_,0.4942,0.4834,0.3664,0.4716,0.4709,0.3512,0.3322,0.4253,0.2838,0.4984,0.4984,0.3316,0.2898,0.3295,0.4827,0.4704,0.3026,0.2572,0.4828,0.4708,0.3293,0.2510\n",
      "ner-ontonotes-_micro_avg_,0.4192,0.3835,0.3631,0.3852,0.3401,0.3371,0.3251,0.3695,0.2918,0.4451,0.3995,0.3406,0.3518,0.3058,0.4156,0.3689,0.2713,0.2670,0.3993,0.3733,0.2694,0.2524\n",
      "srl-conll2012-_clean_micro_,0.3616,0.3563,0.2767,0.3114,0.3051,0.2473,0.2378,0.2920,0.2060,0.3368,0.3280,0.2386,0.2114,0.2312,0.3072,0.2991,0.2136,0.1957,0.3028,0.2947,0.2244,0.1858\n",
      "srl-conll2012-_core_,0.4373,0.4323,0.3279,0.3744,0.3659,0.2821,0.2662,0.3470,0.2198,0.4103,0.3987,0.2715,0.2293,0.2582,0.3697,0.3577,0.2366,0.2080,0.3627,0.3523,0.2539,0.1931\n",
      "srl-conll2012-_non_core_,0.6004,0.5987,0.5049,0.5375,0.5326,0.4939,0.4919,0.5211,0.4595,0.5499,0.5444,0.4787,0.4613,0.4787,0.5289,0.5275,0.4511,0.4389,0.5237,0.5234,0.4557,0.4272\n",
      "coref-ontonotes-conll-1,1.1575,1.0982,1.0452,1.1224,1.0853,1.0394,1.0399,1.0435,0.9293,1.1665,1.1483,0.9587,0.8864,0.9661,1.1189,1.0790,0.7968,0.7565,1.1150,1.0817,0.7732,0.6882\n",
      "spr1-_micro_avg_,1.0894,1.0670,1.0253,1.0467,1.0372,1.0215,1.0259,1.0209,0.9596,1.0781,1.0702,0.9866,0.9811,0.9538,1.0830,1.0729,0.9486,0.9311,1.0839,1.0719,0.9698,0.9398\n",
      "spr2-_micro_avg_,1.1364,1.1320,1.1101,1.1247,1.1216,1.1279,1.1292,1.1167,1.0926,1.1374,1.1407,1.0994,1.0940,1.0948,1.1433,1.1355,1.0878,1.0855,1.1429,1.1389,1.0974,1.0823\n",
      "dpr-_micro_avg_:_mean_,0.3909,,1.0124,1.7884,,,,,1.5782,0.5404,,1.1859,0.4801,,1.4254,,0.9413,2.2944,1.0892,,1.1356,6.0849\n",
      "rel-tacred-_clean_micro_,1.4249,1.4732,1.4733,1.4582,1.4745,,,1.4646,1.4332,1.3576,1.4676,1.4699,1.4692,,1.3324,1.4721,1.4535,1.4326,1.4172,1.4347,1.4553,1.4330\n",
      "rel-semeval-_clean_micro_,3.3007,2.9047,2.6310,3.3899,2.9357,,,3.3301,2.6174,3.2712,2.8558,2.4323,2.4332,,3.3036,2.8974,2.4757,2.3332,3.2739,2.9453,2.5632,2.2861\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sheet_df = fdf.pivot(index=\"display_row\", columns=\"display_col\", values=\"score_errn95\")\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.columns, \n",
    "                                   key=exp_type_sort_key), axis=1)\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.index,\n",
    "                                   key=task_sort_key), axis=0)\n",
    "# sheet_df\n",
    "print((100*sheet_df).to_csv(**csv_args))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot \"final scores\"\n",
    "\n",
    "Make ELMo baselines figure / table."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fdf.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = (fdf.exp_type == \"elmo-chars\") \n",
    "mask |= ((fdf.exp_type == \"elmo-ortho\") | (fdf.exp_type == \"elmo-full\")) & (fdf.tag == \"base\")\n",
    "plot_df = fdf[mask].copy()\n",
    "\n",
    "_SCORE_COL=\"f1_score\"\n",
    "_ERROR_COL=\"f1_errn95\"\n",
    "plot_df['_sort_key'] = plot_df['display_col'].map(task_sort_key)\n",
    "plot_df.sort_values(by=\"_sort_key\", axis=0, inplace=True)\n",
    "plot_df['_display_name'] = [_make_display_name(*tl) \n",
    "                            for tl in zip(plot_df['task'], plot_df['label'])]\n",
    "plot_df['col_key'] = list(zip(plot_df['exp_type'], plot_df['tag']))\n",
    "plot_df['fmt_score'] = plot_df[_SCORE_COL].map(\n",
    "    lambda s: \"{:.0f}\".format(100*s)\n",
    ")\n",
    "plot_df['_err_upper'] = plot_df[_SCORE_COL] + plot_df[_ERROR_COL]\n",
    "plot_df['_err_lower'] = plot_df[_SCORE_COL] - plot_df[_ERROR_COL]\n",
    "print(\"Found %s entries\" % len(plot_df))\n",
    "# long_ds = bokeh.models.ColumnDataSource(data=plot_df)\n",
    "\n",
    "ordered_labels = list(reversed(plot_df['_display_name'].unique())) + [\"\"]\n",
    "_RP = 0.3\n",
    "factor_range = bokeh.models.FactorRange(*ordered_labels, range_padding=_RP,\n",
    "                                        range_padding_units='absolute')\n",
    "tools = \"save,reset\"\n",
    "\n",
    "xstart = 20\n",
    "xend = 100\n",
    "_FONT_SIZE = \"15pt\"\n",
    "p = bp.figure(y_range=factor_range, x_range=[xstart, xend],\n",
    "              height=550, width=1200, tools=tools)\n",
    "\n",
    "label_kw = dict(text_align=\"center\", text_baseline=\"middle\",\n",
    "                text_font_size=_FONT_SIZE)\n",
    "\n",
    "palette = bokeh.palettes.Category20[20]\n",
    "col_keys = [\n",
    "    (\"elmo-chars\", \"base\", \"Lex.\", \"diamond\"),\n",
    "    (\"elmo-chars\", \"cnn1\", \"CNN1\", \"triangle\"),\n",
    "    (\"elmo-chars\", \"cnn2\", \"CNN2\", \"inverted_triangle\"),\n",
    "    (\"elmo-ortho\", \"base\", \"Ortho.\", \"square\"),\n",
    "    (\"elmo-full\", \"base\", \"Full\", \"circle\"),\n",
    "]\n",
    "for i,ck in enumerate(col_keys):\n",
    "    print(ck)\n",
    "    ds = plot_df[plot_df.col_key == ck[:2]]\n",
    "    y = ds[\"_display_name\"]\n",
    "    x = 100*ds[_SCORE_COL]\n",
    "    e = 100*ds[_ERROR_COL]\n",
    "    c = palette[2*i]\n",
    "    cp = palette[2*i+1]\n",
    "    p.hbar(y=y, left=x-e, right=x+e, height=0.90,\n",
    "           fill_color=c, fill_alpha=0.5,\n",
    "           line_color=None, line_width=0)\n",
    "    p.hbar(y=y, left=x-0.05, right=x+0.05, height=0.90,\n",
    "           fill_color=\"Black\", fill_alpha=1.0,\n",
    "           line_color=None, line_width=0)\n",
    "    p.hbar(y=y, left=0, right=x, height=0.90,\n",
    "           fill_color=\"Gray\", fill_alpha=0.05,\n",
    "           line_color=None, line_width=0)\n",
    "    p.scatter(y=y, x=x, size=12, fill_color=c, legend=ck[2],\n",
    "              marker=ck[3])\n",
    "    \n",
    "    dss = bokeh.models.ColumnDataSource(data=ds)\n",
    "    lpos = xstart+(0.7+i)*(xend-xstart)/20\n",
    "    score_labels = bokeh.models.LabelSet(y=\"_display_name\", x=lpos,\n",
    "                                         text=\"fmt_score\", source=dss, **label_kw)\n",
    "    p.add_layout(score_labels)\n",
    "    cat_label = bokeh.models.Label(y=len(ordered_labels)-1+0.6, x=lpos, text=ck[2], \n",
    "                                   angle=np.pi/6, **label_kw)\n",
    "    p.add_layout(cat_label)\n",
    "#     error_bars = bokeh.models.Whisker(base=\"_display_name\", upper=\"_err_upper\", lower=\"_err_lower\",\n",
    "#                                       source=dss, level=\"glyph\", dimension=\"width\")\n",
    "#     p.add_layout(error_bars)\n",
    "    \n",
    "\n",
    "# p.xaxis.major_label_orientation = 1\n",
    "p.xaxis.bounds = (0,100)\n",
    "p.yaxis.bounds = (-_RP, 2*len(col_keys)+2)\n",
    "p.yaxis.major_label_text_font_size = \"13pt\"\n",
    "p.xaxis.major_label_text_font_size = _FONT_SIZE\n",
    "p.ygrid.grid_line_alpha = 0\n",
    "p.xgrid.grid_line_alpha = 0\n",
    "p.xaxis.axis_label = \"F1 Score\"\n",
    "p.xaxis.axis_label_text_font_size = _FONT_SIZE\n",
    "p.legend.orientation = \"horizontal\"\n",
    "p.legend.background_fill_color = None\n",
    "p.legend.border_line_color = None\n",
    "p.legend.label_text_font_size = _FONT_SIZE\n",
    "\n",
    "p.min_border = 0\n",
    "bp.show(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot by constituent height, span distance, etc.\n",
    "\n",
    "Run one of the cells below to populate `plot_df`, and uncomment the corresponding title block."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = (df.split == \"val\")\n",
    "mask &= (df.stratifier == \"info.height\") & (df.task == \"nonterminal-ontonotes\")\n",
    "mask &= (((df['exp_type'] == \"elmo-full\") | (df[\"exp_type\"] == \"elmo-ortho\")) & (df['tag'] == \"base\")) | \\\n",
    "         (df['exp_type'] == \"elmo-chars\")\n",
    "plot_df = df[mask].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = (df.split == \"val\")\n",
    "mask &= df.stratifier.notnull() & (df.task == \"srl-conll2012\")\n",
    "# mask &= ((df['exp_type'] == \"elmo-full\") & (df['tag'] == \"base\")) | \\\n",
    "#          (df['exp_type'] == \"elmo-chars\")\n",
    "mask &= (((df['exp_type'] == \"elmo-full\") | (df[\"exp_type\"] == \"elmo-ortho\")) & (df['tag'] == \"base\")) | \\\n",
    "         (df['exp_type'] == \"elmo-chars\")\n",
    "plot_df = df[mask].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = (df.split == \"val\")\n",
    "mask &= df.stratifier.notnull() & (df.task == \"coref-ontonotes-conll\")\n",
    "# mask &= ((df['exp_type'] == \"elmo-full\") & (df['tag'] == \"base\")) | \\\n",
    "#          (df['exp_type'] == \"elmo-chars\")\n",
    "mask &= (((df['exp_type'] == \"elmo-full\") | (df[\"exp_type\"] == \"elmo-ortho\")) & (df['tag'] == \"base\")) | \\\n",
    "         (df['exp_type'] == \"elmo-chars\")\n",
    "plot_df = df[mask].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = (df.split == \"val\")\n",
    "mask &= df.stratifier.notnull() & (df.task == \"dep-labeling-ewt\")\n",
    "# mask &= ((df['exp_type'] == \"elmo-full\") & (df['tag'] == \"base\")) | \\\n",
    "#          (df['exp_type'] == \"elmo-chars\")\n",
    "mask &= (((df['exp_type'] == \"elmo-full\") | (df[\"exp_type\"] == \"elmo-ortho\")) & (df['tag'] == \"base\")) | \\\n",
    "         (df['exp_type'] == \"elmo-chars\")\n",
    "plot_df = df[mask].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['base', 'bow', 'cnn1', 'cnn2'], dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot_df.tag.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"35cf8d8a-e9de-42ee-911e-fcd3c98e5c78\" data-root-id=\"1003\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"2b9ed80b-2ed8-4c88-bc31-0b2e9e229758\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1012\",\"type\":\"LinearAxis\"}],\"extra_y_ranges\":{\"hist\":{\"id\":\"1203\",\"type\":\"Range1d\"}},\"left\":[{\"id\":\"1017\",\"type\":\"LinearAxis\"}],\"min_border\":0,\"plot_height\":400,\"plot_width\":900,\"renderers\":[{\"id\":\"1012\",\"type\":\"LinearAxis\"},{\"id\":\"1016\",\"type\":\"Grid\"},{\"id\":\"1017\",\"type\":\"LinearAxis\"},{\"id\":\"1021\",\"type\":\"Grid\"},{\"id\":\"1030\",\"type\":\"GlyphRenderer\"},{\"id\":\"1046\",\"type\":\"Legend\"},{\"id\":\"1036\",\"type\":\"GlyphRenderer\"},{\"id\":\"1049\",\"type\":\"Band\"},{\"id\":\"1054\",\"type\":\"GlyphRenderer\"},{\"id\":\"1060\",\"type\":\"GlyphRenderer\"},{\"id\":\"1078\",\"type\":\"Band\"},{\"id\":\"1083\",\"type\":\"GlyphRenderer\"},{\"id\":\"1089\",\"type\":\"GlyphRenderer\"},{\"id\":\"1113\",\"type\":\"Band\"},{\"id\":\"1118\",\"type\":\"GlyphRenderer\"},{\"id\":\"1124\",\"type\":\"GlyphRenderer\"},{\"id\":\"1154\",\"type\":\"Band\"},{\"id\":\"1159\",\"type\":\"GlyphRenderer\"},{\"id\":\"1165\",\"type\":\"GlyphRenderer\"},{\"id\":\"1201\",\"type\":\"Band\"},{\"id\":\"1204\",\"type\":\"LinearAxis\"},{\"id\":\"1208\",\"type\":\"GlyphRenderer\"}],\"right\":[{\"id\":\"1204\",\"type\":\"LinearAxis\"}],\"title\":{\"id\":\"1039\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1024\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"1004\",\"type\":\"Range1d\"},\"x_scale\":{\"id\":\"1008\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1006\",\"type\":\"Range1d\"},\"y_scale\":{\"id\":\"1010\",\"type\":\"LinearScale\"}},\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"marker\":{\"value\":\"inverted_triangle\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1087\",\"type\":\"Scatter\"},{\"attributes\":{},\"id\":\"1075\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1108\",\"type\":\"Selection\"},{\"attributes\":{\"fill_color\":{\"value\":\"Gray\"},\"line_color\":{\"value\":\"#1f77b4\"},\"marker\":{\"value\":\"triangle\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1059\",\"type\":\"Scatter\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1158\",\"type\":\"Line\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"marker\":{\"value\":\"triangle\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1058\",\"type\":\"Scatter\"},{\"attributes\":{},\"id\":\"1022\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"1013\",\"type\":\"BasicTicker\"},{\"attributes\":{\"source\":{\"id\":\"1161\",\"type\":\"ColumnDataSource\"}},\"id\":\"1166\",\"type\":\"CDSView\"},{\"attributes\":{\"data_source\":{\"id\":\"1115\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1116\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1117\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"1119\",\"type\":\"CDSView\"}},\"id\":\"1118\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1051\",\"type\":\"ColumnDataSource\"}},\"id\":\"1055\",\"type\":\"CDSView\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1022\",\"type\":\"SaveTool\"},{\"id\":\"1023\",\"type\":\"ResetTool\"},{\"id\":\"1210\",\"type\":\"HoverTool\"}]},\"id\":\"1024\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1074\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1195\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"axis_label\":\"Span separation distance (tokens)\",\"axis_label_text_font_size\":{\"value\":\"14pt\"},\"formatter\":{\"id\":\"1040\",\"type\":\"BasicTickFormatter\"},\"major_label_text_font_size\":{\"value\":\"14pt\"},\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1013\",\"type\":\"BasicTicker\"}},\"id\":\"1012\",\"type\":\"LinearAxis\"},{\"attributes\":{\"data_source\":{\"id\":\"1056\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1057\",\"type\":\"Scatter\"},\"hover_glyph\":{\"id\":\"1059\",\"type\":\"Scatter\"},\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1058\",\"type\":\"Scatter\"},\"selection_glyph\":null,\"view\":{\"id\":\"1061\",\"type\":\"CDSView\"}},\"id\":\"1060\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"marker\":{\"value\":\"square\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1122\",\"type\":\"Scatter\"},{\"attributes\":{},\"id\":\"1196\",\"type\":\"Selection\"},{\"attributes\":{\"data_source\":{\"id\":\"1051\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1052\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1053\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"1055\",\"type\":\"CDSView\"}},\"id\":\"1054\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1163\",\"type\":\"Scatter\"},{\"attributes\":{},\"id\":\"1044\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"callback\":null,\"end\":1.05,\"start\":0.4},\"id\":\"1006\",\"type\":\"Range1d\"},{\"attributes\":{\"source\":{\"id\":\"1056\",\"type\":\"ColumnDataSource\"}},\"id\":\"1061\",\"type\":\"CDSView\"},{\"attributes\":{\"base\":{\"field\":\"x\",\"units\":\"data\"},\"fill_color\":{\"value\":\"#c5b0d5\"},\"level\":\"underlay\",\"line_color\":{\"value\":\"#9467bd\"},\"lower\":{\"field\":\"lower\",\"units\":\"data\"},\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"source\":{\"id\":\"1200\",\"type\":\"ColumnDataSource\"},\"upper\":{\"field\":\"upper\",\"units\":\"data\"}},\"id\":\"1201\",\"type\":\"Band\"},{\"attributes\":{\"fill_color\":{\"value\":\"Gray\"},\"line_color\":{\"value\":\"#1f77b4\"},\"marker\":{\"value\":\"diamond\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1035\",\"type\":\"Scatter\"},{\"attributes\":{\"fill_color\":{\"value\":\"Gray\"},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1164\",\"type\":\"Scatter\"},{\"attributes\":{\"data_source\":{\"id\":\"1161\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1162\",\"type\":\"Scatter\"},\"hover_glyph\":{\"id\":\"1164\",\"type\":\"Scatter\"},\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1163\",\"type\":\"Scatter\"},\"selection_glyph\":null,\"view\":{\"id\":\"1166\",\"type\":\"CDSView\"}},\"id\":\"1165\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1010\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1023\",\"type\":\"ResetTool\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"XzRVkBTT7T8WptpK/53qP26YRROfUOk/8lNk2Ztg5z8XgsEL7MbmP6+0VlorreU/aoTlnkZY5j9uXd4BR4rmP/FXviSL8ec/41YoboXi5j+AoZqOuUToP8+xRiIgPuo/n+d5nud55j/IpHiBTIrnP9FeQnsJ7eU/YuL9yGm+5j8ZrNAZrNDpPzuxEzuxE+s/WaAFWqAF6j+ZmZmZmZnpP5/ikiFYR+s/AAAAAAAA6D91gynyWTfoP1VVVVVVVe0/2Ymd2Imd6D+bmZmZmZnpP2ijvrNR3+k/z7rBFPms6z+7u7u7u7vrP0dY7mmE5e4/Ka+hvIby6j/UCMs9jbDsPwAAAAAAAPA/VFVVVVVV5T/btm3btm3rPwAAAAAAAPA/AAAAAAAA8D/btm3btm3rPxdddNFFF+0/AAAAAAAA6D8AAAAAAADwP1VVVVVVVeU/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1194\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1193\",\"type\":\"UnionRenderers\"}},\"id\":\"1120\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1193\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"source\":{\"id\":\"1120\",\"type\":\"ColumnDataSource\"}},\"id\":\"1125\",\"type\":\"CDSView\"},{\"attributes\":{\"callback\":null,\"data\":{\"lower\":{\"__ndarray__\":\"Ckvfq3xa7j8THNkBjKDtPylDPVU96uw/xV9PHQ716j/nYoqDPlDqP5fHqkj/C+g/Vb8+BxDa6D/NXtAa3j3nP2BguBl8YOc/5HXqHFeY5z//W59DP5joPz0z2ebIMOo/xr46/J+T5D+9zG8tyovnP2ZF03fW2uE/C5J3O7R45j8GoHTXfFjoPyPICASRDOk/19ri2TnR5T+m4/SlrzvmP0ChvPYvp+c/wXtUAYxw5D9Od0ufwgHdP8k5OT+2Hus/g5UWWHUn4j9a1nueHFniP4k0aex4kec/fNYNpshn7T8K8qqIpqvmP7j0Ls290+k/cG4py1Xj4j9WrX1xudnnP35D3mC3wtw/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P/btm3btm3rPwAAAAAAAPj//Tmf1GjX1D8AAAAAAAD4//05n9Ro19Q/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"upper\":{\"__ndarray__\":\"2B5FBTya7j8X7sUt5gruP/X67QL4i+0/NSdmUkoG7D97g+gvIMXrP+MK7tNdK+o/44EQZJM56z9tnua3zjPqP8KqoK7Gteo/TLiof7FK6z8xMWPN3CLsP3fiZ9yArO0/9GSzjJtO6j9JZ36pr3nsP3ZxmmPgkuk/p0M8CCxO7D92euzhKrntPwWuWSPlVe4/FdjoVIEb7T9epWEybQzuP/4jaUyA5+4/R64StmQ77T8B7wRbySnsP576ttAXe/A/zVau9nad7D+C/YwsITTuP3hPGfm9CvA/fNYNpshn7T+2QmZ36GXwP4JjRvf+8/A/9wJUveC88D8pMgyFsMPwP7wIYsHraPI/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P/btm3btm3rPwAAAAAAAPj/gTHYyiXK8j8AAAAAAAD4/4Ex2MolyvI/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0]},\"selected\":{\"id\":\"1149\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1148\",\"type\":\"UnionRenderers\"}},\"id\":\"1112\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"data\":{\"lower\":{\"__ndarray__\":\"mEkxIb7D7j8AymEwnAfuP5PfTFaas+0/5LMcZ6nw7D+W4yIvKeLrP1lck95+Z+o/4ZmDoMOu6z9hvFLlF0brP58Nl1RCH+s/3ENnFQ2Z6j8QKf2DJ/3rP3WsGTbxGus/07yg+Jtu6T98CwUJ0oXoP25s3DmFJeg/LPvIaV746D/uPXSo5ILpP4WtH4iyaOw/+joUEmXv6T+/5mBRRODsP9MoqOs/JOk/+v4H+9QR5z8+Dgp/Wu3hP2g7cBgLQ+w/RJq8Xa886D/fT8rxjVXnP2jVR0G9p+g/DuU1lNdQ7j/43nvvvffuPwAAAAAAAPj/nud5nud57j9WrX1xudnnPwAAAAAAAPj/AAAAAAAA+P/btm3btm3rPwAAAAAAAPj/AAAAAAAA+P/9OZ/UaNfUP53YiZ3Yie0/6ofYJ7Wxzj8AAAAAAAD4/1VVVVVVVeU/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"upper\":{\"__ndarray__\":\"JvgDRuH67j9g39sLlGnuP9exI86TP+4/fMMuISbG7T9IhyHFlyftP4ucjJEIN+w/w1A8n/uO7T/vqdBow37tP+/t3vSipO0/MLYbqZOW7T+w24AHSanuP7824u0TeO4/YQF5YR+h7T/evD0N4ArtPwyuhH8i7O0/DHdazRMr7j9C+qraDq/uP37HAqDQDPA/WhpBQ/Bl7z/1j8qeslDwP2dlO01Ov+8/ZDwo3Dm67j/C8fWApRLuPwQBTV/MlvA/+uc+qz7n7z9MFFdDdZHwP74ajRikRPA/DuU1lNdQ7j/43nvvvffuPwAAAAAAAPj/nud5nud57j8pMgyFsMPwPwAAAAAAAPj/AAAAAAAA+P/btm3btm3rPwAAAAAAAPj/AAAAAAAA+P+BMdjKJcryP53YiZ3Yie0/V0RasB5/8T8AAAAAAAD4/1VVVVVVVeU/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0]},\"selected\":{\"id\":\"1250\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1249\",\"type\":\"UnionRenderers\"}},\"id\":\"1200\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"6ZbPPtnw7D9KPrD1Yd3oPx/RjvlmN+c/8GezjmoR5T8VvJyCl1PkPxkg8dSQAeI/jWuRoKZC5D+eEuQpQZ7iPyPBQutfxOM/2FBeQ3kN5T9ay9ABuz3mP+iphZWeWug/3VWKCbqr5D9AOpYyCXblP2ApUOtXCuQ/L6l8GPfr4j9JkiRJkiTpP2IndmInduo/nvGMZzzj6T8/+OCDDz7oP1lHm/cpLuk/////////5D/UQR3UQR3kP73pTW960+s/exSuR+F65D93Yid2YifmP1uwBVuwBes/q6qqqqqq6j/UCMs9jbDsPx3HcRzHcew/lpaWlpaW5j9u27Zt27btP5uZmZmZmek/VFVVVVVV5T8AAAAAAADoPwAAAAAAAPA/HcdxHMdx7D/btm3btm3rPxdddNFFF+0/27Zt27Zt6z8AAAAAAADwP1VVVVVVVdU/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1045\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1044\",\"type\":\"UnionRenderers\"}},\"id\":\"1027\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1150\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"XzRVkBTT7T8WptpK/53qP26YRROfUOk/8lNk2Ztg5z8XgsEL7MbmP6+0VlorreU/aoTlnkZY5j9uXd4BR4rmP/FXviSL8ec/41YoboXi5j+AoZqOuUToP8+xRiIgPuo/n+d5nud55j/IpHiBTIrnP9FeQnsJ7eU/YuL9yGm+5j8ZrNAZrNDpPzuxEzuxE+s/WaAFWqAF6j+ZmZmZmZnpP5/ikiFYR+s/AAAAAAAA6D91gynyWTfoP1VVVVVVVe0/2Ymd2Imd6D+bmZmZmZnpP2ijvrNR3+k/z7rBFPms6z+7u7u7u7vrP0dY7mmE5e4/Ka+hvIby6j/UCMs9jbDsPwAAAAAAAPA/VFVVVVVV5T/btm3btm3rPwAAAAAAAPA/AAAAAAAA8D/btm3btm3rPxdddNFFF+0/AAAAAAAA6D8AAAAAAADwP1VVVVVVVeU/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1151\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1150\",\"type\":\"UnionRenderers\"}},\"id\":\"1115\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"1039\",\"type\":\"Title\"},{\"attributes\":{\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1013\",\"type\":\"BasicTicker\"}},\"id\":\"1016\",\"type\":\"Grid\"},{\"attributes\":{\"data_source\":{\"id\":\"1120\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1121\",\"type\":\"Scatter\"},\"hover_glyph\":{\"id\":\"1123\",\"type\":\"Scatter\"},\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1122\",\"type\":\"Scatter\"},\"selection_glyph\":null,\"view\":{\"id\":\"1125\",\"type\":\"CDSView\"}},\"id\":\"1124\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"marker\":{\"value\":\"diamond\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1033\",\"type\":\"Scatter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"marker\":{\"value\":\"diamond\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1034\",\"type\":\"Scatter\"},{\"attributes\":{\"line_color\":\"#9467bd\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1157\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1197\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1198\",\"type\":\"Selection\"},{\"attributes\":{\"fill_color\":{\"value\":\"Gray\"},\"line_color\":{\"value\":\"#1f77b4\"},\"marker\":{\"value\":\"inverted_triangle\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1088\",\"type\":\"Scatter\"},{\"attributes\":{\"fill_color\":{\"value\":\"#d62728\"},\"line_color\":{\"value\":\"#d62728\"},\"marker\":{\"value\":\"square\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1121\",\"type\":\"Scatter\"},{\"attributes\":{\"fill_color\":{\"value\":\"Gray\"},\"line_color\":{\"value\":\"#1f77b4\"},\"marker\":{\"value\":\"square\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1123\",\"type\":\"Scatter\"},{\"attributes\":{},\"id\":\"1109\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1147\",\"type\":\"Selection\"},{\"attributes\":{\"base\":{\"field\":\"x\",\"units\":\"data\"},\"fill_color\":{\"value\":\"#aec7e8\"},\"level\":\"underlay\",\"line_color\":{\"value\":\"#1f77b4\"},\"lower\":{\"field\":\"lower\",\"units\":\"data\"},\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"source\":{\"id\":\"1048\",\"type\":\"ColumnDataSource\"},\"upper\":{\"field\":\"upper\",\"units\":\"data\"}},\"id\":\"1049\",\"type\":\"Band\"},{\"attributes\":{\"label\":{\"value\":\"CNN1\"},\"renderers\":[{\"id\":\"1060\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1076\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1151\",\"type\":\"Selection\"},{\"attributes\":{\"fill_color\":{\"value\":\"#2ca02c\"},\"line_color\":{\"value\":\"#2ca02c\"},\"marker\":{\"value\":\"inverted_triangle\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1086\",\"type\":\"Scatter\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"8TSSWFx67j8Vhc8XudXtPw+fFawaO+0/fcPaN6x96z8xc7lZrwrrPz1pTI6uG+k/nKCntdEJ6j+dfltp1rjoP5GFLGQhC+k/GJdJToRx6T+YRoEIjl3qP9qKoOGk7us/3RF3xB1x5z8DGnfrvALqP27btm3btuU/2erZIXBj6T8+jbDc0wjrPxQ7sRM7ses/dtlll1126T+CRCtsDiTqP5/ikiFYR+s/BJWzW/jV6D9UVVVVVVXlP4OXU/ByCu4/KHZiJ3Zi5z/uaYTlnkboP73pTW960+s/fNYNpshn7T+7u7u7u7vrP97d3d3d3e0/L7rooosu6j/UCMs9jbDsP5uZmZmZmek/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D/btm3btm3rPwAAAAAAAPA/AAAAAAAA6D8AAAAAAADwPwAAAAAAAOg/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1147\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1146\",\"type\":\"UnionRenderers\"}},\"id\":\"1085\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1194\",\"type\":\"Selection\"},{\"attributes\":{\"data_source\":{\"id\":\"1080\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1081\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1082\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"1084\",\"type\":\"CDSView\"}},\"id\":\"1083\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"label\":{\"value\":\"Lex.\"},\"renderers\":[{\"id\":\"1036\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1047\",\"type\":\"LegendItem\"},{\"attributes\":{\"fill_color\":{\"value\":\"#ff7f0e\"},\"line_color\":{\"value\":\"#ff7f0e\"},\"marker\":{\"value\":\"triangle\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1057\",\"type\":\"Scatter\"},{\"attributes\":{\"source\":{\"id\":\"1032\",\"type\":\"ColumnDataSource\"}},\"id\":\"1037\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1146\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"rXIDavyb7j/jN6+R9H3tP95w4tAqjew/18doKf/i6j/lCDxVf5vpP7d8y7d8y+c/ualonlAL6D/XsLJOZmvoPwps2k+WwOY/e492sfdo5z8c6LSBThvoP6zBcwSKtuo/hmEYhmEY5j/vaYTlnkboP+JCzUkgLuQ/45IhWEeb5z8yncV0FtPpPxqt0Rqt0eo/mpmZmZmZ6T/Mli1btmzpP////////+g/AAAAAAAA6T9UVVVVVVXlP4OXU/ByCu4/C9ejcD0K5z98GmG5pxHmPy+66KKLLuo/G8prKK+h7D/UCMs9jbDsP27btm3btu0/zczMzMzM7D+8u7u7u7vrPx3HcRzHcew/AAAAAAAA8D8AAAAAAADoPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADoPwAAAAAAAPA/kyRJkiRJ4j8AAAAAAADwPwAAAAAAAOA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1075\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1074\",\"type\":\"UnionRenderers\"}},\"id\":\"1051\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1249\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"6ZbPPtnw7D9KPrD1Yd3oPx/RjvlmN+c/8GezjmoR5T8VvJyCl1PkPxkg8dSQAeI/jWuRoKZC5D+eEuQpQZ7iPyPBQutfxOM/2FBeQ3kN5T9ay9ABuz3mP+iphZWeWug/3VWKCbqr5D9AOpYyCXblP2ApUOtXCuQ/L6l8GPfr4j9JkiRJkiTpP2IndmInduo/nvGMZzzj6T8/+OCDDz7oP1lHm/cpLuk/////////5D/UQR3UQR3kP73pTW960+s/exSuR+F65D93Yid2YifmP1uwBVuwBes/q6qqqqqq6j/UCMs9jbDsPx3HcRzHcew/lpaWlpaW5j9u27Zt27btP5uZmZmZmek/VFVVVVVV5T8AAAAAAADoPwAAAAAAAPA/HcdxHMdx7D/btm3btm3rPxdddNFFF+0/27Zt27Zt6z8AAAAAAADwP1VVVVVVVdU/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1071\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1070\",\"type\":\"UnionRenderers\"}},\"id\":\"1032\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1008\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1107\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"line_color\":\"#d62728\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1116\",\"type\":\"Line\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1018\",\"type\":\"BasicTicker\"}},\"id\":\"1021\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1045\",\"type\":\"Selection\"},{\"attributes\":{\"axis_label\":\"F1 Score\",\"axis_label_text_font_size\":{\"value\":\"14pt\"},\"bounds\":[0,1],\"formatter\":{\"id\":\"1042\",\"type\":\"BasicTickFormatter\"},\"major_label_text_font_size\":{\"value\":\"13pt\"},\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1018\",\"type\":\"BasicTicker\"}},\"id\":\"1017\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1149\",\"type\":\"Selection\"},{\"attributes\":{\"base\":{\"field\":\"x\",\"units\":\"data\"},\"fill_color\":{\"value\":\"#98df8a\"},\"level\":\"underlay\",\"line_color\":{\"value\":\"#2ca02c\"},\"lower\":{\"field\":\"lower\",\"units\":\"data\"},\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"source\":{\"id\":\"1112\",\"type\":\"ColumnDataSource\"},\"upper\":{\"field\":\"upper\",\"units\":\"data\"}},\"id\":\"1113\",\"type\":\"Band\"},{\"attributes\":{},\"id\":\"1247\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1110\",\"type\":\"Selection\"},{\"attributes\":{\"label\":{\"value\":\"Full\"},\"renderers\":[{\"id\":\"1165\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1199\",\"type\":\"LegendItem\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1117\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1042\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1106\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1040\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1072\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"source\":{\"id\":\"1115\",\"type\":\"ColumnDataSource\"}},\"id\":\"1119\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1105\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"source\":{\"id\":\"1080\",\"type\":\"ColumnDataSource\"}},\"id\":\"1084\",\"type\":\"CDSView\"},{\"attributes\":{\"base\":{\"field\":\"x\",\"units\":\"data\"},\"fill_color\":{\"value\":\"#ff9896\"},\"level\":\"underlay\",\"line_color\":{\"value\":\"#d62728\"},\"lower\":{\"field\":\"lower\",\"units\":\"data\"},\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"source\":{\"id\":\"1153\",\"type\":\"ColumnDataSource\"},\"upper\":{\"field\":\"upper\",\"units\":\"data\"}},\"id\":\"1154\",\"type\":\"Band\"},{\"attributes\":{\"data_source\":{\"id\":\"1027\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1028\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1029\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"1031\",\"type\":\"CDSView\"}},\"id\":\"1030\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"line_color\":\"#ff7f0e\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1052\",\"type\":\"Line\"},{\"attributes\":{\"source\":{\"id\":\"1156\",\"type\":\"ColumnDataSource\"}},\"id\":\"1160\",\"type\":\"CDSView\"},{\"attributes\":{\"data_source\":{\"id\":\"1156\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1157\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1158\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"1160\",\"type\":\"CDSView\"}},\"id\":\"1159\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1053\",\"type\":\"Line\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"rXIDavyb7j/jN6+R9H3tP95w4tAqjew/18doKf/i6j/lCDxVf5vpP7d8y7d8y+c/ualonlAL6D/XsLJOZmvoPwps2k+WwOY/e492sfdo5z8c6LSBThvoP6zBcwSKtuo/hmEYhmEY5j/vaYTlnkboP+JCzUkgLuQ/45IhWEeb5z8yncV0FtPpPxqt0Rqt0eo/mpmZmZmZ6T/Mli1btmzpP////////+g/AAAAAAAA6T9UVVVVVVXlP4OXU/ByCu4/C9ejcD0K5z98GmG5pxHmPy+66KKLLuo/G8prKK+h7D/UCMs9jbDsP27btm3btu0/zczMzMzM7D+8u7u7u7vrPx3HcRzHcew/AAAAAAAA8D8AAAAAAADoPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADoPwAAAAAAAPA/kyRJkiRJ4j8AAAAAAADwPwAAAAAAAOA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1106\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1105\",\"type\":\"UnionRenderers\"}},\"id\":\"1056\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1018\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1071\",\"type\":\"Selection\"},{\"attributes\":{\"data_source\":{\"id\":\"1085\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1086\",\"type\":\"Scatter\"},\"hover_glyph\":{\"id\":\"1088\",\"type\":\"Scatter\"},\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1087\",\"type\":\"Scatter\"},\"selection_glyph\":null,\"view\":{\"id\":\"1090\",\"type\":\"CDSView\"}},\"id\":\"1089\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"base\":{\"field\":\"x\",\"units\":\"data\"},\"fill_color\":{\"value\":\"#ffbb78\"},\"level\":\"underlay\",\"line_color\":{\"value\":\"#ff7f0e\"},\"lower\":{\"field\":\"lower\",\"units\":\"data\"},\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"source\":{\"id\":\"1077\",\"type\":\"ColumnDataSource\"},\"upper\":{\"field\":\"upper\",\"units\":\"data\"}},\"id\":\"1078\",\"type\":\"Band\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1082\",\"type\":\"Line\"},{\"attributes\":{\"data_source\":{\"id\":\"1032\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1033\",\"type\":\"Scatter\"},\"hover_glyph\":{\"id\":\"1035\",\"type\":\"Scatter\"},\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1034\",\"type\":\"Scatter\"},\"selection_glyph\":null,\"view\":{\"id\":\"1037\",\"type\":\"CDSView\"}},\"id\":\"1036\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"36Cas0/f7j+w1B4emDjuP7VIOBKX+e0/sLslxGdb7T9vNSJ64ITsP3L8D7hDT+s/UvXfn9+e7D8osxGnbWLsP8f9uqTyYew/Bn1BX9AX7D9gAr9FOFPtP5rx/ZGCyew/Gt8Mrd2H6z8tZCELWcjqPz2NsNzTCOs/HLmRG7mR6z8YnI/B+RjsP0GeEuQpQe4/qqqqqqqq7D9UA3vH1MDuPx3HcRzHcew/rx2Yawfm6j8AAAAAAADoP7gehetRuO4/H8F9BPcR7D88PDw8PDzsP3IFMbmCmOw/DuU1lNdQ7j/43nvvvffuPwAAAAAAAPA/nud5nud57j/UCMs9jbDsPwAAAAAAAPA/AAAAAAAA8D/btm3btm3rPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADoP53YiZ3Yie0/VFVVVVVV5T8AAAAAAADwP1VVVVVVVeU/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1248\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1247\",\"type\":\"UnionRenderers\"}},\"id\":\"1161\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"label\":{\"value\":\"CNN2\"},\"renderers\":[{\"id\":\"1089\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1111\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1073\",\"type\":\"Selection\"},{\"attributes\":{\"label\":{\"value\":\"Ortho.\"},\"renderers\":[{\"id\":\"1124\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1152\",\"type\":\"LegendItem\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"8TSSWFx67j8Vhc8XudXtPw+fFawaO+0/fcPaN6x96z8xc7lZrwrrPz1pTI6uG+k/nKCntdEJ6j+dfltp1rjoP5GFLGQhC+k/GJdJToRx6T+YRoEIjl3qP9qKoOGk7us/3RF3xB1x5z8DGnfrvALqP27btm3btuU/2erZIXBj6T8+jbDc0wjrPxQ7sRM7ses/dtlll1126T+CRCtsDiTqP5/ikiFYR+s/BJWzW/jV6D9UVVVVVVXlP4OXU/ByCu4/KHZiJ3Zi5z/uaYTlnkboP73pTW960+s/fNYNpshn7T+7u7u7u7vrP97d3d3d3e0/L7rooosu6j/UCMs9jbDsP5uZmZmZmek/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D/btm3btm3rPwAAAAAAAPA/AAAAAAAA6D8AAAAAAADwPwAAAAAAAOg/AAAAAAAA8D8AAAAAAADwPwAAAAAAAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1110\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1109\",\"type\":\"UnionRenderers\"}},\"id\":\"1080\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"data\":{\"lower\":{\"__ndarray__\":\"x4CW/6Su7T9iEJkEiU/qPxOtyhQn2ug/B+T2T8ut5j9l2lmpVtTlP4dwvepzceQ/zRH0BzTh5D8U78un7t/kP1JlnrxEMOY/0IjuFhm65D8TuvJWvizmPyUQ8ziXF+g/YKTwncmA4z8HsYSZSsXkP4GZYiErGOI/zXf/TSQ74z9R8fERR67mP1sk2hMkLug/cnyMULyQ5j+NwJJ6KZHlP0ChvPYvp+c/Uee7re1+4z+Gk32vjh7iP0mhTQTjG+o/IMNsyrrE4z+ZnvPmaQvkP774R1gKzOQ/42ZcHzRO5z8K8qqIpqvmP0dY7mmE5e4/Ka+hvIby6j9WrX1xudnnPwAAAAAAAPj/6ofYJ7Wxzj/btm3btm3rPwAAAAAAAPj/AAAAAAAA+P/btm3btm3rPxdddNFFF+0//Tmf1GjX1D8AAAAAAAD4/1VVVVVVVeU/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"upper\":{\"__ndarray__\":\"9+cTIYT37T/KOxyRdezqP8mDwBEXx+k/3cPRYmwT6D/JKSlugbnnP9f478ni6OY/B/fWNVnP5z/Iy/BbnzToP5BK3ozRsuk/9iRixfEK6T/tiELGtFzqP3lTmgupZOw/3ioDnwVz6T+JmGxpTk/qPyEkItXnwek/90z8Q69B6j/hZq8hEfPsPxs+TWI++e0/QMR+Y4R67T+lcqC4CaLtP/4jaUyA5+4/rxhEUhKB7D9kc9U0JVDuP7CELtNjR/A/klDO5lh27T+dlD9MySfvPxJONQ+Z8u4/XocTBd8F8D+2QmZ36GXwP0dY7mmE5e4/Ka+hvIby6j8pMgyFsMPwPwAAAAAAAPj/V0RasB5/8T/btm3btm3rPwAAAAAAAPj/AAAAAAAA+P/btm3btm3rPxdddNFFF+0/gTHYyiXK8j8AAAAAAAD4/1VVVVVVVeU/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0]},\"selected\":{\"id\":\"1196\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1195\",\"type\":\"UnionRenderers\"}},\"id\":\"1153\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0],\"y\":{\"__ndarray__\":\"36Cas0/f7j+w1B4emDjuP7VIOBKX+e0/sLslxGdb7T9vNSJ64ITsP3L8D7hDT+s/UvXfn9+e7D8osxGnbWLsP8f9uqTyYew/Bn1BX9AX7D9gAr9FOFPtP5rx/ZGCyew/Gt8Mrd2H6z8tZCELWcjqPz2NsNzTCOs/HLmRG7mR6z8YnI/B+RjsP0GeEuQpQe4/qqqqqqqq7D9UA3vH1MDuPx3HcRzHcew/rx2Yawfm6j8AAAAAAADoP7gehetRuO4/H8F9BPcR7D88PDw8PDzsP3IFMbmCmOw/DuU1lNdQ7j/43nvvvffuPwAAAAAAAPA/nud5nud57j/UCMs9jbDsPwAAAAAAAPA/AAAAAAAA8D/btm3btm3rPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADoP53YiZ3Yie0/VFVVVVVV5T8AAAAAAADwP1VVVVVVVeU/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"dtype\":\"float64\",\"shape\":[56]}},\"selected\":{\"id\":\"1198\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1197\",\"type\":\"UnionRenderers\"}},\"id\":\"1156\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"source\":{\"id\":\"1085\",\"type\":\"ColumnDataSource\"}},\"id\":\"1090\",\"type\":\"CDSView\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1029\",\"type\":\"Line\"},{\"attributes\":{\"line_color\":\"#2ca02c\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1081\",\"type\":\"Line\"},{\"attributes\":{\"fill_color\":{\"value\":\"#9467bd\"},\"line_color\":{\"value\":\"#9467bd\"},\"size\":{\"units\":\"screen\",\"value\":12},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1162\",\"type\":\"Scatter\"},{\"attributes\":{\"source\":{\"id\":\"1027\",\"type\":\"ColumnDataSource\"}},\"id\":\"1031\",\"type\":\"CDSView\"},{\"attributes\":{\"callback\":null,\"data\":{\"lower\":{\"__ndarray__\":\"tjm4bW197j+FcW2Mx0TtP60NvsMtNOw/5AUScbFS6j9uA2pBysnoP4//4qstqeY/MTAxsQut5j+dvbnCxujmP2AwVizI3uQ/nYN/6XlM5T9+CtOdBvzlP6FGEDajxeg/jm2OUB8S4z9SSUDjM6blP7K43MSnKeA/yx4G3NI65D/8WuKsn9TmP6kOAWJOyec/cIT1hgwP5j/ctRvZRUXlP1rg/80ocOQ//bF50VLW5D9Od0ufwgHdP8k5OT+2Hus/jl9UgPXa4T8IvrB7MPHdP8DB5oTYZeU/vWnbqlaX6D/UCMs9jbDsP27btm3btu0/zczMzMzM7D/gFCVKdTrmPx3HcRzHcew/AAAAAAAA+P/9OZ/UaNfUPwAAAAAAAPj/AAAAAAAA+P/9OZ/UaNfUPwAAAAAAAPj/MKt1FbEKrz8AAAAAAAD4/4AUrkfheoQ/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"upper\":{\"__ndarray__\":\"pKtOZou67j9B/vCWIbftPw/UBt4n5uw/yom/4Uxz6z9cDg5pNG3qP9/5s8PL7eg/QSOgi5Vp6T8RpKvaBe7pP7SnXnNkoug/WZtteXWF6T+6xZZlljrqP7c819Jwp+w/flWiu6Me6T+MisjnCefqPxLNvc6YMug/+wY91Lv76j9o36g8jdHsP4tLotML2u0/xK49rCYk7T+8dz/dJpTtP6QfADLXj+0/A06GLq0p7T8B7wRbySnsP576ttAXe/A/iE7zYIU57D/01ek0tyrtP56y6sA+9+4/PBX+0gNW8D/UCMs9jbDsP27btm3btu0/zczMzMzM7D9MMakWgZ7wPx3HcRzHcew/AAAAAAAA+P+BMdjKJcryPwAAAAAAAPj/AAAAAAAA+P+BMdjKJcryPwAAAAAAAPj/OnedCc9Q8T8AAAAAAAD4/65H4XoUru8/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0]},\"selected\":{\"id\":\"1108\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1107\",\"type\":\"UnionRenderers\"}},\"id\":\"1077\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"background_fill_color\":{\"value\":null},\"border_line_color\":{\"value\":null},\"items\":[{\"id\":\"1047\",\"type\":\"LegendItem\"},{\"id\":\"1076\",\"type\":\"LegendItem\"},{\"id\":\"1111\",\"type\":\"LegendItem\"},{\"id\":\"1152\",\"type\":\"LegendItem\"},{\"id\":\"1199\",\"type\":\"LegendItem\"}],\"label_text_font_size\":{\"value\":\"14pt\"},\"orientation\":\"horizontal\",\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"1046\",\"type\":\"Legend\"},{\"attributes\":{\"callback\":null,\"data\":{\"lower\":{\"__ndarray__\":\"Bx4V8f/G7D/bhELxE4boP42/3Q+/tOY/nyhjk6RP5D9pr7MU+kzjPzZDrB5Xp+A/ZK820wmw4j/zqNpqt8PgP0QFZlN6t+E/9IKLJpLH4j+6EwL39vrjP3vK+N32++U/6q7z6EeF4T+Sa0TxCHTiP4UhEMOuD+A/ax4cw3P13T9kFAqiFgrmP0dFTVdgW+c/kHgS4th95j8p561FRBPkPx7V3CQetOQ/VB8Xasi23z/WcBLsqhfaP8ZyaVKBI+g/DriENa1W3T93Yid2YifmPyauyNhOfeY/xysB5cj+5T/UCMs9jbDsPx3HcRzHcew/lpaWlpaW5j9u27Zt27btP35D3mC3wtw/6ofYJ7Wxzj/9OZ/UaNfUPwAAAAAAAPj/HcdxHMdx7D/btm3btm3rPxdddNFFF+0/27Zt27Zt6z8AAAAAAAD4/1qwKhhvtci/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"upper\":{\"__ndarray__\":\"yw+KjLIa7T+59x36rzTpP7HiP+MOuuc/QacDijDT5T/ByIXwNFrlP/z8NYvKW+M/tifsbUPV5T9JfO3oynjkPwJ9H4NF0eU/vB4xYGBT5z/6gp8Mf4DoP1WJEk1Gueo/0PwgKizS5z/uCOhzCXjoPzsxkBMBBeg/KENrTzTd5j8uED/wDT/sP30Jn23ukO0/rGoH7Z9I7T9VCRTC2mjsP5S5Wco1qO0/VHD0ypsk6j89SzEyri7rP7RgMoxzg+8/78yZ9GtK6j93Yid2YifmP5CyQt0Rju8/jylUcIxW7z/UCMs9jbDsPx3HcRzHcew/lpaWlpaW5j9u27Zt27btP7wIYsHraPI/V0RasB5/8T+BMdjKJcryPwAAAAAAAPj/HcdxHMdx7D/btm3btm3rPxdddNFFF+0/27Zt27Zt6z8AAAAAAAD4/2wBYBuxgus/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/wAAAAAAAPj/AAAAAAAA+P8AAAAAAAD4/w==\",\"dtype\":\"float64\",\"shape\":[56]},\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0]},\"selected\":{\"id\":\"1073\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1072\",\"type\":\"UnionRenderers\"}},\"id\":\"1048\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"end\":15.5,\"start\":-0.5},\"id\":\"1004\",\"type\":\"Range1d\"},{\"attributes\":{\"data_source\":{\"id\":\"1205\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1206\",\"type\":\"VBar\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1207\",\"type\":\"VBar\"},\"selection_glyph\":null,\"view\":{\"id\":\"1209\",\"type\":\"CDSView\"},\"y_range_name\":\"hist\"},\"id\":\"1208\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"top\"},\"width\":{\"value\":0.9},\"x\":{\"field\":\"x\"}},\"id\":\"1207\",\"type\":\"VBar\"},{\"attributes\":{},\"id\":\"1250\",\"type\":\"Selection\"},{\"attributes\":{\"source\":{\"id\":\"1205\",\"type\":\"ColumnDataSource\"}},\"id\":\"1209\",\"type\":\"CDSView\"},{\"attributes\":{\"fill_color\":{\"value\":\"Gray\"},\"line_color\":{\"value\":\"Gray\"},\"top\":{\"field\":\"top\"},\"width\":{\"value\":0.9},\"x\":{\"field\":\"x\"}},\"id\":\"1206\",\"type\":\"VBar\"},{\"attributes\":{\"callback\":null,\"end\":37456.66666666667,\"start\":1.0},\"id\":\"1203\",\"type\":\"Range1d\"},{\"attributes\":{\"axis_label_text_font_size\":{\"value\":\"14pt\"},\"bounds\":[1.0,11237.0],\"formatter\":{\"id\":\"1212\",\"type\":\"NumeralTickFormatter\"},\"major_label_text_font_size\":{\"value\":\"13pt\"},\"plot\":{\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1220\",\"type\":\"BasicTicker\"},\"y_range_name\":\"hist\"},\"id\":\"1204\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1252\",\"type\":\"Selection\"},{\"attributes\":{\"callback\":null,\"renderers\":[{\"id\":\"1036\",\"type\":\"GlyphRenderer\"},{\"id\":\"1060\",\"type\":\"GlyphRenderer\"},{\"id\":\"1089\",\"type\":\"GlyphRenderer\"},{\"id\":\"1124\",\"type\":\"GlyphRenderer\"},{\"id\":\"1165\",\"type\":\"GlyphRenderer\"}],\"tooltips\":[[\"Height\",\"@x\"],[\"F1 score\",\"@y{0.00}\"]]},\"id\":\"1210\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1070\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1148\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1251\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1248\",\"type\":\"Selection\"},{\"attributes\":{\"callback\":null,\"data\":{\"top\":{\"__ndarray__\":\"AAAAAIDyxUAAAAAAAGi1QAAAAAAAOqdAAAAAAABQmUAAAAAAABiNQAAAAAAAuIJAAAAAAADweEAAAAAAAEBzQAAAAAAA4G1AAAAAAACAZkAAAAAAAKBkQAAAAAAAgF9AAAAAAAAAWEAAAAAAAMBYQAAAAAAAgE5AAAAAAAAAUUAAAAAAAIBPQAAAAAAAAExAAAAAAACASEAAAAAAAABDQAAAAAAAgEBAAAAAAAAAQkAAAAAAAAA0QAAAAAAAADlAAAAAAAAAPUAAAAAAAAAxQAAAAAAAADhAAAAAAAAANEAAAAAAAAAwQAAAAAAAAC5AAAAAAAAAJkAAAAAAAAAuQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAQQAAAAAAAAAhAAAAAAAAAFEAAAAAAAAAQQAAAAAAAABhAAAAAAAAAEEAAAAAAAAAAQAAAAAAAABBAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAABAAAAAAAAACEAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAACA8sVAAAAAAABotUAAAAAAADqnQAAAAAAAUJlAAAAAAAAYjUAAAAAAALiCQAAAAAAA8HhAAAAAAABAc0AAAAAAAOBtQAAAAAAAgGZAAAAAAACgZEAAAAAAAIBfQAAAAAAAAFhAAAAAAADAWEAAAAAAAIBOQAAAAAAAAFFAAAAAAACAT0AAAAAAAABMQAAAAAAAgEhAAAAAAAAAQ0AAAAAAAIBAQAAAAAAAAEJAAAAAAAAANEAAAAAAAAA5QAAAAAAAAD1AAAAAAAAAMUAAAAAAAAA4QAAAAAAAADRAAAAAAAAAMEAAAAAAAAAuQAAAAAAAACZAAAAAAAAALkAAAAAAAAAUQAAAAAAAABRAAAAAAAAAEEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAEEAAAAAAAAAYQAAAAAAAABBAAAAAAAAAAEAAAAAAAAAQQAAAAAAAAABAAAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAAhAAAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAgPLFQAAAAAAAaLVAAAAAAAA6p0AAAAAAAFCZQAAAAAAAGI1AAAAAAAC4gkAAAAAAAPB4QAAAAAAAQHNAAAAAAADgbUAAAAAAAIBmQAAAAAAAoGRAAAAAAACAX0AAAAAAAABYQAAAAAAAwFhAAAAAAACATkAAAAAAAABRQAAAAAAAgE9AAAAAAAAATEAAAAAAAIBIQAAAAAAAAENAAAAAAACAQEAAAAAAAABCQAAAAAAAADRAAAAAAAAAOUAAAAAAAAA9QAAAAAAAADFAAAAAAAAAOEAAAAAAAAA0QAAAAAAAADBAAAAAAAAALkAAAAAAAAAmQAAAAAAAAC5AAAAAAAAAFEAAAAAAAAAUQAAAAAAAABBAAAAAAAAACEAAAAAAAAAUQAAAAAAAABBAAAAAAAAAGEAAAAAAAAAQQAAAAAAAAABAAAAAAAAAEEAAAAAAAAAAQAAAAAAAAABAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAIDyxUAAAAAAAGi1QAAAAAAAOqdAAAAAAABQmUAAAAAAABiNQAAAAAAAuIJAAAAAAADweEAAAAAAAEBzQAAAAAAA4G1AAAAAAACAZkAAAAAAAKBkQAAAAAAAgF9AAAAAAAAAWEAAAAAAAMBYQAAAAAAAgE5AAAAAAAAAUUAAAAAAAIBPQAAAAAAAAExAAAAAAACASEAAAAAAAABDQAAAAAAAgEBAAAAAAAAAQkAAAAAAAAA0QAAAAAAAADlAAAAAAAAAPUAAAAAAAAAxQAAAAAAAADhAAAAAAAAANEAAAAAAAAAwQAAAAAAAAC5AAAAAAAAAJkAAAAAAAAAuQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAQQAAAAAAAAAhAAAAAAAAAFEAAAAAAAAAQQAAAAAAAABhAAAAAAAAAEEAAAAAAAAAAQAAAAAAAABBAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAABAAAAAAAAACEAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAACA8sVAAAAAAABotUAAAAAAADqnQAAAAAAAUJlAAAAAAAAYjUAAAAAAALiCQAAAAAAA8HhAAAAAAABAc0AAAAAAAOBtQAAAAAAAgGZAAAAAAACgZEAAAAAAAIBfQAAAAAAAAFhAAAAAAADAWEAAAAAAAIBOQAAAAAAAAFFAAAAAAACAT0AAAAAAAABMQAAAAAAAgEhAAAAAAAAAQ0AAAAAAAIBAQAAAAAAAAEJAAAAAAAAANEAAAAAAAAA5QAAAAAAAAD1AAAAAAAAAMUAAAAAAAAA4QAAAAAAAADRAAAAAAAAAMEAAAAAAAAAuQAAAAAAAACZAAAAAAAAALkAAAAAAAAAUQAAAAAAAABRAAAAAAAAAEEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAEEAAAAAAAAAYQAAAAAAAABBAAAAAAAAAAEAAAAAAAAAQQAAAAAAAAABAAAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAAhAAAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAgPLFQAAAAAAAaLVAAAAAAAA6p0AAAAAAAFCZQAAAAAAAGI1AAAAAAAC4gkAAAAAAAPB4QAAAAAAAQHNAAAAAAADgbUAAAAAAAIBmQAAAAAAAoGRAAAAAAACAX0AAAAAAAABYQAAAAAAAwFhAAAAAAACATkAAAAAAAABRQAAAAAAAgE9AAAAAAAAATEAAAAAAAIBIQAAAAAAAAENAAAAAAACAQEAAAAAAAABCQAAAAAAAADRAAAAAAAAAOUAAAAAAAAA9QAAAAAAAADFAAAAAAAAAOEAAAAAAAAA0QAAAAAAAADBAAAAAAAAALkAAAAAAAAAmQAAAAAAAAC5AAAAAAAAAFEAAAAAAAAAUQAAAAAAAABBAAAAAAAAACEAAAAAAAAAUQAAAAAAAABBAAAAAAAAAGEAAAAAAAAAQQAAAAAAAAABAAAAAAAAAEEAAAAAAAAAAQAAAAAAAAABAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/\",\"dtype\":\"float64\",\"shape\":[336]},\"x\":[0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0,0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0,0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0,0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0,0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0,0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,51.0,54.0,56.0,57.0,60.0,61.0,71.0]},\"selected\":{\"id\":\"1252\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1251\",\"type\":\"UnionRenderers\"}},\"id\":\"1205\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"format\":\"0a\"},\"id\":\"1212\",\"type\":\"NumeralTickFormatter\"},{\"attributes\":{\"line_color\":\"#1f77b4\",\"line_width\":2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1028\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1220\",\"type\":\"BasicTicker\"}],\"root_ids\":[\"1003\"]},\"title\":\"Bokeh Application\",\"version\":\"1.0.4\"}};\n",
       "  var render_items = [{\"docid\":\"2b9ed80b-2ed8-4c88-bc31-0b2e9e229758\",\"roots\":{\"1003\":\"35cf8d8a-e9de-42ee-911e-fcd3c98e5c78\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "1003"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "height = 400\n",
    "width = 900\n",
    "_FONT_SIZE = \"14pt\"\n",
    "\n",
    "# title = \"F1 by height on OntoNotes constituent labeling\"\n",
    "# _X_LABEL = \"Constituent Height\"\n",
    "# _X_RANGE = [1.5,20.5]\n",
    "# _Y_RANGE = [0,1]\n",
    "# title = \"F1 by span distance on OntoNotes SRL\"\n",
    "# _X_LABEL = \"Span separation distance (tokens)\"\n",
    "# _X_RANGE = [-0.5, 15.5]\n",
    "# _Y_RANGE = [0.5,1]\n",
    "# _X_LABEL = \"Span separation distance (tokens)\"\n",
    "# _X_RANGE = [-0.5, 15.5]\n",
    "# _Y_RANGE = [0.5,1]\n",
    "# title = \"F1 by span distance on dependency labeling\"\n",
    "_X_LABEL = \"Span separation distance (tokens)\"\n",
    "_X_RANGE = [-0.5, 15.5]\n",
    "_Y_RANGE = [0.4,1.05]\n",
    "\n",
    "# def _make_plot_legend_name(exp_type, tag):\n",
    "#     if exp_type == \"elmo-full\":\n",
    "#         return \"Full ELMo\"\n",
    "#     elif exp_type == \"elmo-chars\":\n",
    "#         return f\"Lex. ({tag})\"\n",
    "#     else:\n",
    "#         raise ValueError(f\"Unrecognized experiment: ({exp_type}, {tag})\")\n",
    "\n",
    "col_keys = [\n",
    "    (\"elmo-chars\", \"base\", \"Lex.\", \"diamond\"),\n",
    "    (\"elmo-chars\", \"cnn1\", \"CNN1\", \"triangle\"),\n",
    "    (\"elmo-chars\", \"cnn2\", \"CNN2\", \"inverted_triangle\"),\n",
    "    (\"elmo-ortho\", \"base\", \"Ortho.\", \"square\"),\n",
    "    (\"elmo-full\", \"base\", \"Full\", \"circle\"),\n",
    "]\n",
    "\n",
    "plot_df['col_key'] = list(zip(plot_df['exp_type'], plot_df['tag']))\n",
    "        \n",
    "_SCORE_COL=\"f1_score\"\n",
    "_ERROR_COL=\"f1_errn95\"\n",
    "\n",
    "tools = \"save,reset\"\n",
    "\n",
    "palette = bokeh.palettes.Category20[20]\n",
    "p = bp.figure(x_range=_X_RANGE, y_range=_Y_RANGE, \n",
    "              width=width, height=height,\n",
    "              tools=tools)\n",
    "crs = []\n",
    "# gb = plot_df.groupby(by=[\"exp_type\", \"tag\", \"display_col\"])\n",
    "# for i, ((exp_type, tag, name), idx) in enumerate(gb.groups.items()):\n",
    "for i, ck in enumerate(col_keys):\n",
    "    ds = plot_df[plot_df['col_key'] == ck[:2]]\n",
    "    exp_type = ck[0]\n",
    "    et_key = exp_type + (f\"-{tag}\" if tag != \"base\" else \"\")\n",
    "#     x = plot_df.loc[idx, \"stratum_key\"]\n",
    "#     y = plot_df.loc[idx, _SCORE_COL]\n",
    "#     e = plot_df.loc[idx, _ERROR_COL]\n",
    "    x = ds[\"stratum_key\"]\n",
    "    y = ds[_SCORE_COL]\n",
    "    e = ds[_ERROR_COL]\n",
    "    c = palette[2*i]\n",
    "#     display_name = _make_plot_legend_name(exp_type, tag) + \"  \"\n",
    "#     display_name = ck[2]\n",
    "    p.line(x=x, y=y, color=c, line_width=2)\n",
    "    cr = p.scatter(x=x, y=y, color=c, size=12, hover_fill_color=\"Gray\",\n",
    "                   legend=ck[2], marker=ck[3])\n",
    "    crs.append(cr)\n",
    "    ds = bokeh.models.ColumnDataSource(data=dict(x=x, upper=y+e, lower=y-e))\n",
    "#     error_bars = bokeh.models.Whisker(base='x', upper='upper', lower='lower',\n",
    "#                                       line_color=c,\n",
    "#                                       source=ds)\n",
    "#     p.add_layout(error_bars)\n",
    "    error_band = bokeh.models.Band(base='x', upper='upper', lower='lower',\n",
    "                                   source=ds, level='underlay',\n",
    "                                   fill_alpha=0.4, line_width=1,\n",
    "                                   line_color=palette[2*i],\n",
    "                                   fill_color=palette[2*i+1])\n",
    "    p.add_layout(error_band)\n",
    "\n",
    "p.yaxis.bounds = (0,1)\n",
    "\n",
    "##\n",
    "# Overlay histogram at bottom of the plot (this looks nice)\n",
    "counts = plot_df['tp_count'] + plot_df['fn_count']\n",
    "strata = plot_df['stratum_key']\n",
    "hist_height = 0.3\n",
    "p.extra_y_ranges = {\"hist\": bokeh.models.Range1d(min(counts), \n",
    "                                                 max(counts)/hist_height)}\n",
    "p.add_layout(bokeh.models.LinearAxis(y_range_name=\"hist\",\n",
    "                                     bounds=(min(counts), max(counts))), \"right\")\n",
    "p.vbar(x=strata, top=counts, width=0.9,\n",
    "       y_range_name='hist', color=\"Gray\")\n",
    "\n",
    "# Add fancy hover tool\n",
    "tooltips = [\n",
    "    (\"Height\", \"@x\"),\n",
    "    (\"F1 score\", \"@y{0.00}\"),\n",
    "]\n",
    "p.add_tools(bokeh.models.HoverTool(tooltips=tooltips, renderers=crs, \n",
    "#                                    mode='vline',\n",
    "                                  ))\n",
    "p.xaxis.axis_label = _X_LABEL\n",
    "p.yaxis[0].axis_label = \"F1 Score\"\n",
    "p.yaxis[1].axis_label = \"\"\n",
    "p.yaxis[1].formatter = bokeh.models.NumeralTickFormatter(format=\"0a\")\n",
    "p.legend.orientation = \"horizontal\"\n",
    "# p.legend.location = \"bottom_right\"\n",
    "# p.legend.background_fill_alpha = 0.0\n",
    "# p.legend.border_line_alpha = 0\n",
    "\n",
    "p.yaxis.major_label_text_font_size = \"13pt\"\n",
    "p.xaxis.major_label_text_font_size = \"13pt\"\n",
    "p.xaxis.major_label_text_font_size = _FONT_SIZE\n",
    "p.xaxis.axis_label_text_font_size = _FONT_SIZE\n",
    "p.yaxis.axis_label_text_font_size = _FONT_SIZE\n",
    "# p.legend.orientation = \"horizontal\"\n",
    "p.legend.background_fill_color = None\n",
    "p.legend.border_line_color = None\n",
    "p.legend.label_text_font_size = _FONT_SIZE\n",
    "\n",
    "p.min_border = 0\n",
    "\n",
    "bp.show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"2311\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"2311\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    }\n",
       "    finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        root._bokeh_is_loading--;\n",
       "        if (root._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"2311\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '2311' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.0.4.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"2311\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"2311\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n    }\n    finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.info(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(js_urls, callback) {\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = js_urls.length;\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var s = document.createElement('script');\n      s.src = url;\n      s.async = false;\n      s.onreadystatechange = s.onload = function() {\n        root._bokeh_is_loading--;\n        if (root._bokeh_is_loading === 0) {\n          console.log(\"Bokeh: all BokehJS libraries loaded\");\n          run_callbacks()\n        }\n      };\n      s.onerror = function() {\n        console.warn(\"failed to load library \" + url);\n      };\n      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.getElementsByTagName(\"head\")[0].appendChild(s);\n    }\n  };var element = document.getElementById(\"2311\");\n  if (element == null) {\n    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '2311' but no matching script tag was found. \")\n    return false;\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.0.4.min.js\"];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.4.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.4.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.4.min.css\");\n    }\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"2311\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(js_urls, function() {\n      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import bokeh\n",
    "import bokeh.plotting as bp\n",
    "bp.output_notebook()\n",
    "\n",
    "import datetime\n",
    "import socket\n",
    "def get_compact_timestamp():\n",
    "    now = datetime.datetime.now()\n",
    "    return now.strftime(\"%Y%m%d.%H%M%S\")\n",
    "\n",
    "def _save_figure_to_bucket(fig, name, title=None, export_format=\"html\"):\n",
    "    now = get_compact_timestamp()\n",
    "    fname = f\"{name}.{now:s}.{export_format}\"\n",
    "    title = title or name\n",
    "    if fname.endswith('.png'):\n",
    "        bokeh.io.export_png(p, os.path.join(\"/tmp\", fname))\n",
    "    else:\n",
    "        bp.save(p, os.path.join(\"/tmp\", fname), title=title, \n",
    "                resources=bokeh.resources.CDN)\n",
    "    hostname = socket.gethostname()\n",
    "    !gsutil cp /tmp/$fname gs://edge-probing/$hostname/plots/$fname\n",
    "    !gsutil acl ch -u AllUsers:R gs://edge-probing/$hostname/plots/$fname\n",
    "    url = f\"https://storage.googleapis.com/edge-probing/{hostname}/plots/{fname}\"\n",
    "    print(f\"Public URL: {url}\")\n",
    "    return url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file:///tmp/scores_by_distance.20190504.001735.html [Content-Type=text/html]...\n",
      "/ [1 files][ 44.8 KiB/ 44.8 KiB]                                                \n",
      "Operation completed over 1 objects/44.8 KiB.                                     \n",
      "Updated ACL on gs://edge-probing/iftenney/plots/scores_by_distance.20190504.001735.html\n",
      "Public URL: https://storage.googleapis.com/edge-probing/iftenney/plots/scores_by_distance.20190504.001735.html\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://storage.googleapis.com/edge-probing/iftenney/plots/scores_by_distance.20190504.001735.html'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_save_figure_to_bucket(p, name=\"scores_by_distance\",\n",
    "                       title=\"Scores by distance\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p1 = p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mask = (df.split == \"val\")\n",
    "mask &= (df.stratifier == \"info.height\") & (df.task == \"constituent-ontonotes\")\n",
    "mask &= df['exp_type'].map(lambda t: t in {'elmo-chars', 'elmo-full', 'elmo-ortho', 'openai-cat'})\n",
    "sheet_df = df[mask].pivot(index=\"stratum_key\", columns=\"display_col\", values=\"f1_score\")\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.columns, \n",
    "                                   key=exp_type_sort_key), axis=1)\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.index), axis=0)\n",
    "# sheet_df\n",
    "print(sheet_df.to_csv())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sheet_df = df[mask].pivot(index=\"stratum_key\", columns=\"display_col\", values=\"f1_errn95\")\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.columns, \n",
    "                                   key=exp_type_sort_key), axis=1)\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.index), axis=0)\n",
    "# sheet_df\n",
    "print(sheet_df.to_csv())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot by span distance\n",
    "\n",
    "Absolute distance (in number of tokens under model tokenization) between midpoint of `span1` and midpoint of `span2`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['task'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = df.stratifier.notnull() & (df.task == \"srl-conll2012\") & (df.split == \"val\")\n",
    "mask &= df['exp_type'].map(lambda t: t in {'elmo-chars', 'elmo-full', 'elmo-ortho', 'openai-cat'})\n",
    "sheet_df = df[mask].pivot(index=\"stratum_key\", columns=\"display_col\", values=\"f1_score\")\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.columns, \n",
    "                                   key=exp_type_sort_key), axis=1)\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.index), axis=0)\n",
    "# sheet_df\n",
    "print(sheet_df.to_csv())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot by label for Relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = (df.task == \"rel-semeval\") & (df.split == \"val\")\n",
    "mask &= df.label.map(is_positive_relation)\n",
    "sheet_df = df[mask].pivot(index=\"label\", columns=\"exp_type\", values=\"f1_score\")\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.columns, \n",
    "                                   key=exp_type_sort_key), axis=1)\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.index,\n",
    "                                   key=task_sort_key), axis=0)\n",
    "# sheet_df\n",
    "print(sheet_df.to_csv())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot by label for SRL core, non-core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = df.label.map(is_core_or_noncore) & (df.task == \"srl-conll2012\") & (df.split == \"val\")\n",
    "sheet_df = df[mask].pivot(index=\"label\", columns=\"exp_type\", values=\"f1_score\")\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.columns, \n",
    "                                   key=exp_type_sort_key), axis=1)\n",
    "sheet_df = sheet_df.reindex(sorted(sheet_df.index,\n",
    "                                   key=task_sort_key), axis=0)\n",
    "# sheet_df\n",
    "print(sheet_df.to_csv())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[mask].exp_type.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_df = df[df.label.map(is_core_role) & (df.task == \"srl-conll2012\")].copy()\n",
    "\n",
    "_SCORE_COL=\"f1_score\"\n",
    "plot_df['row_key'] = list(zip(plot_df['task'], plot_df['exp_type']))\n",
    "plot_df['fmt_score'] = plot_df[_SCORE_COL].map(\n",
    "    lambda s: \"{:.02f}\".format(s)\n",
    ")\n",
    "print(\"Found %s entries\" % len(plot_df))\n",
    "long_ds = bokeh.models.ColumnDataSource(data=plot_df)\n",
    "\n",
    "factor_range = bokeh.models.FactorRange(*categories, range_padding=0.5,\n",
    "                                       range_padding_units='absolute')\n",
    "tools = \"save,reset\"\n",
    "\n",
    "p = bp.figure(x_range=factor_range, y_range=[0,1],\n",
    "              width=1250, tools=tools)\n",
    "p.vbar(x='row_key', top=_SCORE_COL, width=0.95,\n",
    "       fill_color=fill_cmap,\n",
    "       line_color=\"Gray\", source=long_ds)\n",
    "label_kw = dict(text_align=\"right\", text_baseline=\"middle\", y_offset=-3,\n",
    "                text_font_size=\"11pt\", angle=90, angle_units='deg')\n",
    "score_labels = bokeh.models.LabelSet(x='row_key', y=_SCORE_COL,\n",
    "                                     text=\"fmt_score\",\n",
    "                                     source=long_ds, **label_kw)\n",
    "p.add_layout(score_labels)\n",
    "p.xaxis.major_label_orientation = 1\n",
    "p.yaxis.bounds = (0,1)\n",
    "\n",
    "bp.show(p)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "jiant",
   "language": "python",
   "name": "jiant"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
